fix(ollama): report finish_reason length when ollama/ hits max_tokens

The ollama/ provider (the /api/generate route) always set finish_reason to
stop and never read Ollama's done_reason. A reply cut off by max_tokens looked
complete, and /v1/messages answered end_turn instead of max_tokens.

Read done_reason on the non-streaming and streaming paths, the same way
ollama_chat/ already does. A missing or null done_reason still means stop, and
a JSON-mode reply that parses into a function call still reports tool_calls.
This commit is contained in:
gjija 2026-11-05 02:38:00 +00:00
parent 78f1b09429
commit eab84e6ac5
No known key found for this signature in database
2 changed files with 68 additions and 9 deletions

View file

@ -11,6 +11,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
@ -96,6 +97,7 @@ class _OllamaGenerateResponse(TypedDict):
message: ReadOnly[NotRequired[_OllamaGenerateMessage]]
prompt_eval_count: ReadOnly[NotRequired[int]]
eval_count: ReadOnly[NotRequired[int]]
done_reason: ReadOnly[NotRequired[str | None]]
_OLLAMA_GENERATE_RESPONSE: Final = TypeAdapter(_OllamaGenerateResponse)
@ -321,7 +323,7 @@ class OllamaConfig(BaseConfig):
) -> ModelResponse:
response_json: Final = _OLLAMA_GENERATE_RESPONSE.validate_python(raw_response.json())
## RESPONSE OBJECT
model_response.choices[0].finish_reason = "stop"
model_response.choices[0].finish_reason = map_finish_reason(response_json.get("done_reason") or "stop")
if request_data.get("format", "") == "json":
# Check if response field exists and is not empty before parsing JSON
response_text = response_json.get("response", "")
@ -331,7 +333,6 @@ class OllamaConfig(BaseConfig):
# Handle empty response gracefully - set empty content
message = litellm.Message(content="", reasoning_content=thinking)
model_response.choices[0].message = message
model_response.choices[0].finish_reason = "stop"
else:
try:
response_content: Final = _JSON_VALUE.validate_python(
@ -369,13 +370,11 @@ class OllamaConfig(BaseConfig):
reasoning_content=thinking,
)
model_response.choices[0].message = message
model_response.choices[0].finish_reason = "stop"
except json.JSONDecodeError:
# If JSON parsing fails, treat as regular text response
reasoning_content, content = _OllamaGenerateReasoning.from_response(response_json).split()
message = litellm.Message(content=content, reasoning_content=reasoning_content)
model_response.choices[0].message = message
model_response.choices[0].finish_reason = "stop"
else:
reasoning_content, content = _OllamaGenerateReasoning.from_response(response_json).split()
model_response.choices[0].message.content = content
@ -541,13 +540,13 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
self.held_content = ""
return released
def _flush_held_content(self, usage: ChatCompletionUsageBlock | None) -> GenericStreamingChunk:
def _flush_held_content(self, usage: ChatCompletionUsageBlock | None, finish_reason: str) -> GenericStreamingChunk:
held: Final = self.held_content
self.held_content = ""
try:
parsed: Final = _JSON_VALUE.validate_python(json.loads(_strip_json_code_fence(held)))
except json.JSONDecodeError:
return GenericStreamingChunk(text=held, is_finished=True, finish_reason="stop", usage=usage)
return GenericStreamingChunk(text=held, is_finished=True, finish_reason=finish_reason, usage=usage)
if isinstance(parsed, dict) and "name" in parsed and "arguments" in parsed:
return GenericStreamingChunk(
text="",
@ -563,7 +562,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
finish_reason="tool_calls",
usage=usage,
)
return GenericStreamingChunk(text=held, is_finished=True, finish_reason="stop", usage=usage)
return GenericStreamingChunk(text=held, is_finished=True, finish_reason=finish_reason, usage=usage)
def _handle_string_chunk(self, str_line: str) -> GenericStreamingChunk | ModelResponseStream:
return self.chunk_parser(json.loads(str_line))
@ -579,7 +578,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
if chunk["done"] is True:
text = ""
is_finished = True
finish_reason = "stop"
finish_reason = _OLLAMA_GENERATE_RESPONSE.validate_python(chunk).get("done_reason") or "stop"
prompt_eval_count: Final[int | None] = chunk.get("prompt_eval_count", None)
eval_count: Final[int | None] = chunk.get("eval_count", None)
@ -591,7 +590,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
total_tokens=prompt_eval_count + eval_count,
)
if self.held_content:
return self._flush_held_content(usage)
return self._flush_held_content(usage, finish_reason)
return GenericStreamingChunk(
text=text,
is_finished=is_finished,

View file

@ -597,6 +597,39 @@ class TestOllamaConfig:
assert result.choices[0]["message"].content == ""
assert result.choices[0]["finish_reason"] == "stop"
@pytest.mark.parametrize(
("response_text", "request_data", "done_reason", "expected_finish_reason"),
[
("one two three", None, "length", "length"),
('{"answer": "one two', {"format": "json"}, "length", "length"),
('{"answer": 42}', {"format": "json"}, "length", "length"),
("", {"format": "json"}, "length", "length"),
('{"name": "get_weather", "arguments": {}}', {"format": "json"}, "length", "tool_calls"),
("one two three", None, None, "stop"),
],
ids=[
"text",
"json_mode_cut_off",
"json_mode_valid_json",
"json_mode_empty",
"json_mode_function_call",
"null_done_reason",
],
)
def test_transform_response_finish_reason_follows_done_reason(
self,
response_text: str,
request_data: dict[str, object] | None,
done_reason: str | None,
expected_finish_reason: str,
):
result = self._transform(
{"response": response_text, "done": True, "done_reason": done_reason, "eval_count": 3},
request_data=request_data,
)
assert result.choices[0]["finish_reason"] == expected_finish_reason
class TestOllamaTextCompletionResponseIterator:
def test_every_chunk_of_one_stream_carries_the_same_response_id(self):
@ -767,6 +800,33 @@ class TestOllamaTextCompletionResponseIterator:
assert (first.choices[0].delta.content, second.choices[0].delta.content) == ("Here is code: ", '{"a": 1}')
assert (done["text"], done["finish_reason"], done.get("tool_use")) == ("", "stop", None)
@pytest.mark.parametrize(
("response_chunks", "done_reason", "expected_text", "expected_finish_reason"),
[
(("one ", "two "), "length", "", "length"),
(('{"answer": ', '"one two'), "length", '{"answer": "one two', "length"),
(('{"name": "get_weather", ', '"arguments": {}}'), "length", "", "tool_calls"),
(("one ", "two "), None, "", "stop"),
],
ids=["text", "held_json_cut_off", "held_function_call", "null_done_reason"],
)
def test_chunk_parser_finish_reason_follows_done_reason(
self,
response_chunks: tuple[str, ...],
done_reason: str | None,
expected_text: str,
expected_finish_reason: str,
):
iterator: Final = OllamaTextCompletionResponseIterator(
streaming_response=iter([]), sync_stream=True, json_mode=False
)
for text in response_chunks:
iterator.chunk_parser({"response": text, "done": False})
done: Final = iterator.chunk_parser({"response": "", "done": True, "done_reason": done_reason})
assert (done["text"], done["finish_reason"]) == (expected_text, expected_finish_reason)
def test_chunk_parser_releases_held_code_fence_that_is_not_json(self):
iterator: Final = OllamaTextCompletionResponseIterator(
streaming_response=iter([]), sync_stream=True, json_mode=False