From eab84e6ac55462865bab8844d3f490c0328a2f65 Mon Sep 17 00:00:00 2001 From: gjija Date: Thu, 5 Nov 2026 02:38:00 +0000 Subject: [PATCH] fix(ollama): report finish_reason length when ollama/ hits max_tokens The ollama/ provider (the /api/generate route) always set finish_reason to stop and never read Ollama's done_reason. A reply cut off by max_tokens looked complete, and /v1/messages answered end_turn instead of max_tokens. Read done_reason on the non-streaming and streaming paths, the same way ollama_chat/ already does. A missing or null done_reason still means stop, and a JSON-mode reply that parses into a function call still reports tool_calls. --- .../llms/ollama/completion/transformation.py | 17 +++--- .../test_ollama_completion_transformation.py | 60 +++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 6dbcb050e11..199ca73f4ce 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -11,6 +11,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) @@ -96,6 +97,7 @@ class _OllamaGenerateResponse(TypedDict): message: ReadOnly[NotRequired[_OllamaGenerateMessage]] prompt_eval_count: ReadOnly[NotRequired[int]] eval_count: ReadOnly[NotRequired[int]] + done_reason: ReadOnly[NotRequired[str | None]] _OLLAMA_GENERATE_RESPONSE: Final = TypeAdapter(_OllamaGenerateResponse) @@ -321,7 +323,7 @@ class OllamaConfig(BaseConfig): ) -> ModelResponse: response_json: Final = _OLLAMA_GENERATE_RESPONSE.validate_python(raw_response.json()) ## RESPONSE OBJECT - model_response.choices[0].finish_reason = "stop" + model_response.choices[0].finish_reason = map_finish_reason(response_json.get("done_reason") or "stop") if request_data.get("format", "") == "json": # Check if response field exists and is not empty before parsing JSON response_text = response_json.get("response", "") @@ -331,7 +333,6 @@ class OllamaConfig(BaseConfig): # Handle empty response gracefully - set empty content message = litellm.Message(content="", reasoning_content=thinking) model_response.choices[0].message = message - model_response.choices[0].finish_reason = "stop" else: try: response_content: Final = _JSON_VALUE.validate_python( @@ -369,13 +370,11 @@ class OllamaConfig(BaseConfig): reasoning_content=thinking, ) model_response.choices[0].message = message - model_response.choices[0].finish_reason = "stop" except json.JSONDecodeError: # If JSON parsing fails, treat as regular text response reasoning_content, content = _OllamaGenerateReasoning.from_response(response_json).split() message = litellm.Message(content=content, reasoning_content=reasoning_content) model_response.choices[0].message = message - model_response.choices[0].finish_reason = "stop" else: reasoning_content, content = _OllamaGenerateReasoning.from_response(response_json).split() model_response.choices[0].message.content = content @@ -541,13 +540,13 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): self.held_content = "" return released - def _flush_held_content(self, usage: ChatCompletionUsageBlock | None) -> GenericStreamingChunk: + def _flush_held_content(self, usage: ChatCompletionUsageBlock | None, finish_reason: str) -> GenericStreamingChunk: held: Final = self.held_content self.held_content = "" try: parsed: Final = _JSON_VALUE.validate_python(json.loads(_strip_json_code_fence(held))) except json.JSONDecodeError: - return GenericStreamingChunk(text=held, is_finished=True, finish_reason="stop", usage=usage) + return GenericStreamingChunk(text=held, is_finished=True, finish_reason=finish_reason, usage=usage) if isinstance(parsed, dict) and "name" in parsed and "arguments" in parsed: return GenericStreamingChunk( text="", @@ -563,7 +562,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): finish_reason="tool_calls", usage=usage, ) - return GenericStreamingChunk(text=held, is_finished=True, finish_reason="stop", usage=usage) + return GenericStreamingChunk(text=held, is_finished=True, finish_reason=finish_reason, usage=usage) def _handle_string_chunk(self, str_line: str) -> GenericStreamingChunk | ModelResponseStream: return self.chunk_parser(json.loads(str_line)) @@ -579,7 +578,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): if chunk["done"] is True: text = "" is_finished = True - finish_reason = "stop" + finish_reason = _OLLAMA_GENERATE_RESPONSE.validate_python(chunk).get("done_reason") or "stop" prompt_eval_count: Final[int | None] = chunk.get("prompt_eval_count", None) eval_count: Final[int | None] = chunk.get("eval_count", None) @@ -591,7 +590,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): total_tokens=prompt_eval_count + eval_count, ) if self.held_content: - return self._flush_held_content(usage) + return self._flush_held_content(usage, finish_reason) return GenericStreamingChunk( text=text, is_finished=is_finished, diff --git a/tests/unit/llms/ollama/test_ollama_completion_transformation.py b/tests/unit/llms/ollama/test_ollama_completion_transformation.py index 26addf1b24c..79a01c99c8f 100644 --- a/tests/unit/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_completion_transformation.py @@ -597,6 +597,39 @@ class TestOllamaConfig: assert result.choices[0]["message"].content == "" assert result.choices[0]["finish_reason"] == "stop" + @pytest.mark.parametrize( + ("response_text", "request_data", "done_reason", "expected_finish_reason"), + [ + ("one two three", None, "length", "length"), + ('{"answer": "one two', {"format": "json"}, "length", "length"), + ('{"answer": 42}', {"format": "json"}, "length", "length"), + ("", {"format": "json"}, "length", "length"), + ('{"name": "get_weather", "arguments": {}}', {"format": "json"}, "length", "tool_calls"), + ("one two three", None, None, "stop"), + ], + ids=[ + "text", + "json_mode_cut_off", + "json_mode_valid_json", + "json_mode_empty", + "json_mode_function_call", + "null_done_reason", + ], + ) + def test_transform_response_finish_reason_follows_done_reason( + self, + response_text: str, + request_data: dict[str, object] | None, + done_reason: str | None, + expected_finish_reason: str, + ): + result = self._transform( + {"response": response_text, "done": True, "done_reason": done_reason, "eval_count": 3}, + request_data=request_data, + ) + + assert result.choices[0]["finish_reason"] == expected_finish_reason + class TestOllamaTextCompletionResponseIterator: def test_every_chunk_of_one_stream_carries_the_same_response_id(self): @@ -767,6 +800,33 @@ class TestOllamaTextCompletionResponseIterator: assert (first.choices[0].delta.content, second.choices[0].delta.content) == ("Here is code: ", '{"a": 1}') assert (done["text"], done["finish_reason"], done.get("tool_use")) == ("", "stop", None) + @pytest.mark.parametrize( + ("response_chunks", "done_reason", "expected_text", "expected_finish_reason"), + [ + (("one ", "two "), "length", "", "length"), + (('{"answer": ', '"one two'), "length", '{"answer": "one two', "length"), + (('{"name": "get_weather", ', '"arguments": {}}'), "length", "", "tool_calls"), + (("one ", "two "), None, "", "stop"), + ], + ids=["text", "held_json_cut_off", "held_function_call", "null_done_reason"], + ) + def test_chunk_parser_finish_reason_follows_done_reason( + self, + response_chunks: tuple[str, ...], + done_reason: str | None, + expected_text: str, + expected_finish_reason: str, + ): + iterator: Final = OllamaTextCompletionResponseIterator( + streaming_response=iter([]), sync_stream=True, json_mode=False + ) + + for text in response_chunks: + iterator.chunk_parser({"response": text, "done": False}) + done: Final = iterator.chunk_parser({"response": "", "done": True, "done_reason": done_reason}) + + assert (done["text"], done["finish_reason"]) == (expected_text, expected_finish_reason) + def test_chunk_parser_releases_held_code_fence_that_is_not_json(self): iterator: Final = OllamaTextCompletionResponseIterator( streaming_response=iter([]), sync_stream=True, json_mode=False