mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(ollama): report finish_reason length when ollama/ hits max_tokens
The ollama/ provider (the /api/generate route) always set finish_reason to stop and never read Ollama's done_reason. A reply cut off by max_tokens looked complete, and /v1/messages answered end_turn instead of max_tokens. Read done_reason on the non-streaming and streaming paths, the same way ollama_chat/ already does. A missing or null done_reason still means stop, and a JSON-mode reply that parses into a function call still reports tool_calls.
This commit is contained in:
parent
78f1b09429
commit
eab84e6ac5
2 changed files with 68 additions and 9 deletions
|
|
@ -11,6 +11,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_str_from_messages,
|
||||
)
|
||||
|
|
@ -96,6 +97,7 @@ class _OllamaGenerateResponse(TypedDict):
|
|||
message: ReadOnly[NotRequired[_OllamaGenerateMessage]]
|
||||
prompt_eval_count: ReadOnly[NotRequired[int]]
|
||||
eval_count: ReadOnly[NotRequired[int]]
|
||||
done_reason: ReadOnly[NotRequired[str | None]]
|
||||
|
||||
|
||||
_OLLAMA_GENERATE_RESPONSE: Final = TypeAdapter(_OllamaGenerateResponse)
|
||||
|
|
@ -321,7 +323,7 @@ class OllamaConfig(BaseConfig):
|
|||
) -> ModelResponse:
|
||||
response_json: Final = _OLLAMA_GENERATE_RESPONSE.validate_python(raw_response.json())
|
||||
## RESPONSE OBJECT
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
model_response.choices[0].finish_reason = map_finish_reason(response_json.get("done_reason") or "stop")
|
||||
if request_data.get("format", "") == "json":
|
||||
# Check if response field exists and is not empty before parsing JSON
|
||||
response_text = response_json.get("response", "")
|
||||
|
|
@ -331,7 +333,6 @@ class OllamaConfig(BaseConfig):
|
|||
# Handle empty response gracefully - set empty content
|
||||
message = litellm.Message(content="", reasoning_content=thinking)
|
||||
model_response.choices[0].message = message
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
else:
|
||||
try:
|
||||
response_content: Final = _JSON_VALUE.validate_python(
|
||||
|
|
@ -369,13 +370,11 @@ class OllamaConfig(BaseConfig):
|
|||
reasoning_content=thinking,
|
||||
)
|
||||
model_response.choices[0].message = message
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
except json.JSONDecodeError:
|
||||
# If JSON parsing fails, treat as regular text response
|
||||
reasoning_content, content = _OllamaGenerateReasoning.from_response(response_json).split()
|
||||
message = litellm.Message(content=content, reasoning_content=reasoning_content)
|
||||
model_response.choices[0].message = message
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
else:
|
||||
reasoning_content, content = _OllamaGenerateReasoning.from_response(response_json).split()
|
||||
model_response.choices[0].message.content = content
|
||||
|
|
@ -541,13 +540,13 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
|
|||
self.held_content = ""
|
||||
return released
|
||||
|
||||
def _flush_held_content(self, usage: ChatCompletionUsageBlock | None) -> GenericStreamingChunk:
|
||||
def _flush_held_content(self, usage: ChatCompletionUsageBlock | None, finish_reason: str) -> GenericStreamingChunk:
|
||||
held: Final = self.held_content
|
||||
self.held_content = ""
|
||||
try:
|
||||
parsed: Final = _JSON_VALUE.validate_python(json.loads(_strip_json_code_fence(held)))
|
||||
except json.JSONDecodeError:
|
||||
return GenericStreamingChunk(text=held, is_finished=True, finish_reason="stop", usage=usage)
|
||||
return GenericStreamingChunk(text=held, is_finished=True, finish_reason=finish_reason, usage=usage)
|
||||
if isinstance(parsed, dict) and "name" in parsed and "arguments" in parsed:
|
||||
return GenericStreamingChunk(
|
||||
text="",
|
||||
|
|
@ -563,7 +562,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
|
|||
finish_reason="tool_calls",
|
||||
usage=usage,
|
||||
)
|
||||
return GenericStreamingChunk(text=held, is_finished=True, finish_reason="stop", usage=usage)
|
||||
return GenericStreamingChunk(text=held, is_finished=True, finish_reason=finish_reason, usage=usage)
|
||||
|
||||
def _handle_string_chunk(self, str_line: str) -> GenericStreamingChunk | ModelResponseStream:
|
||||
return self.chunk_parser(json.loads(str_line))
|
||||
|
|
@ -579,7 +578,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
|
|||
if chunk["done"] is True:
|
||||
text = ""
|
||||
is_finished = True
|
||||
finish_reason = "stop"
|
||||
finish_reason = _OLLAMA_GENERATE_RESPONSE.validate_python(chunk).get("done_reason") or "stop"
|
||||
prompt_eval_count: Final[int | None] = chunk.get("prompt_eval_count", None)
|
||||
eval_count: Final[int | None] = chunk.get("eval_count", None)
|
||||
|
||||
|
|
@ -591,7 +590,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
|
|||
total_tokens=prompt_eval_count + eval_count,
|
||||
)
|
||||
if self.held_content:
|
||||
return self._flush_held_content(usage)
|
||||
return self._flush_held_content(usage, finish_reason)
|
||||
return GenericStreamingChunk(
|
||||
text=text,
|
||||
is_finished=is_finished,
|
||||
|
|
|
|||
|
|
@ -597,6 +597,39 @@ class TestOllamaConfig:
|
|||
assert result.choices[0]["message"].content == ""
|
||||
assert result.choices[0]["finish_reason"] == "stop"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("response_text", "request_data", "done_reason", "expected_finish_reason"),
|
||||
[
|
||||
("one two three", None, "length", "length"),
|
||||
('{"answer": "one two', {"format": "json"}, "length", "length"),
|
||||
('{"answer": 42}', {"format": "json"}, "length", "length"),
|
||||
("", {"format": "json"}, "length", "length"),
|
||||
('{"name": "get_weather", "arguments": {}}', {"format": "json"}, "length", "tool_calls"),
|
||||
("one two three", None, None, "stop"),
|
||||
],
|
||||
ids=[
|
||||
"text",
|
||||
"json_mode_cut_off",
|
||||
"json_mode_valid_json",
|
||||
"json_mode_empty",
|
||||
"json_mode_function_call",
|
||||
"null_done_reason",
|
||||
],
|
||||
)
|
||||
def test_transform_response_finish_reason_follows_done_reason(
|
||||
self,
|
||||
response_text: str,
|
||||
request_data: dict[str, object] | None,
|
||||
done_reason: str | None,
|
||||
expected_finish_reason: str,
|
||||
):
|
||||
result = self._transform(
|
||||
{"response": response_text, "done": True, "done_reason": done_reason, "eval_count": 3},
|
||||
request_data=request_data,
|
||||
)
|
||||
|
||||
assert result.choices[0]["finish_reason"] == expected_finish_reason
|
||||
|
||||
|
||||
class TestOllamaTextCompletionResponseIterator:
|
||||
def test_every_chunk_of_one_stream_carries_the_same_response_id(self):
|
||||
|
|
@ -767,6 +800,33 @@ class TestOllamaTextCompletionResponseIterator:
|
|||
assert (first.choices[0].delta.content, second.choices[0].delta.content) == ("Here is code: ", '{"a": 1}')
|
||||
assert (done["text"], done["finish_reason"], done.get("tool_use")) == ("", "stop", None)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("response_chunks", "done_reason", "expected_text", "expected_finish_reason"),
|
||||
[
|
||||
(("one ", "two "), "length", "", "length"),
|
||||
(('{"answer": ', '"one two'), "length", '{"answer": "one two', "length"),
|
||||
(('{"name": "get_weather", ', '"arguments": {}}'), "length", "", "tool_calls"),
|
||||
(("one ", "two "), None, "", "stop"),
|
||||
],
|
||||
ids=["text", "held_json_cut_off", "held_function_call", "null_done_reason"],
|
||||
)
|
||||
def test_chunk_parser_finish_reason_follows_done_reason(
|
||||
self,
|
||||
response_chunks: tuple[str, ...],
|
||||
done_reason: str | None,
|
||||
expected_text: str,
|
||||
expected_finish_reason: str,
|
||||
):
|
||||
iterator: Final = OllamaTextCompletionResponseIterator(
|
||||
streaming_response=iter([]), sync_stream=True, json_mode=False
|
||||
)
|
||||
|
||||
for text in response_chunks:
|
||||
iterator.chunk_parser({"response": text, "done": False})
|
||||
done: Final = iterator.chunk_parser({"response": "", "done": True, "done_reason": done_reason})
|
||||
|
||||
assert (done["text"], done["finish_reason"]) == (expected_text, expected_finish_reason)
|
||||
|
||||
def test_chunk_parser_releases_held_code_fence_that_is_not_json(self):
|
||||
iterator: Final = OllamaTextCompletionResponseIterator(
|
||||
streaming_response=iter([]), sync_stream=True, json_mode=False
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue