diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 0fc1cd926b8..b1f69220de7 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -4,6 +4,7 @@ from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Any, Final from httpx._models import Headers, Response +from pydantic import BaseModel, ConfigDict, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -43,6 +44,37 @@ else: LiteLLMLoggingObj = Any +class _OllamaGenerateReasoning(BaseModel): + """The two `/api/generate` fields a reply's reasoning can arrive in.""" + + model_config = ConfigDict(extra="ignore") + + # Absent and explicitly null are distinct here: Ollama omits `response` where it sends + # no text, and sends null where the reply carries none, which stay "" and None downstream. + response: str | None = "" + thinking: str | None = None + + @classmethod + def from_response(cls, response_json: object) -> "_OllamaGenerateReasoning": + try: + return cls.model_validate(response_json) + except ValidationError: + return cls() + + def split(self) -> tuple[str | None, str | None]: + """Reasoning reaches `/api/generate` either in the top-level `thinking` field or + inline in `` tags, never both. The field wins, matching `ollama_chat`.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _parse_content_for_reasoning, + ) + + if self.thinking: + return self.thinking, self.response + if self.response is None: + return None, None + return _parse_content_for_reasoning(self.response) + + class OllamaConfig(BaseConfig): """ Reference: https://github.com/ollama/ollama/blob/main/docs/api.md#parameters @@ -255,20 +287,17 @@ class OllamaConfig(BaseConfig): api_key: str | None = None, json_mode: bool | None = None, ) -> ModelResponse: - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - _parse_content_for_reasoning, - ) - response_json: Final = raw_response.json() ## RESPONSE OBJECT model_response.choices[0].finish_reason = "stop" if request_data.get("format", "") == "json": # Check if response field exists and is not empty before parsing JSON response_text = response_json.get("response", "") + thinking: Final = _OllamaGenerateReasoning.from_response(response_json).thinking or None if not response_text or not response_text.strip(): # Handle empty response gracefully - set empty content - message = litellm.Message(content="") + message = litellm.Message(content="", reasoning_content=thinking) model_response.choices[0].message = message model_response.choices[0].finish_reason = "stop" else: @@ -285,6 +314,7 @@ class OllamaConfig(BaseConfig): function_call: Final = response_content message = litellm.Message( content=None, + reasoning_content=thinking, tool_calls=[ { "id": f"call_{uuid.uuid4()}", @@ -302,27 +332,18 @@ class OllamaConfig(BaseConfig): # Handle as regular JSON (new behavior) message = litellm.Message( content=json.dumps(response_content), + reasoning_content=thinking, ) model_response.choices[0].message = message model_response.choices[0].finish_reason = "stop" except json.JSONDecodeError: # If JSON parsing fails, treat as regular text response - ## output parse reasoning content from response_text - reasoning_content: str | None = None - content: str | None = None - if response_text is not None: - reasoning_content, content = _parse_content_for_reasoning(response_text) + reasoning_content, content = _OllamaGenerateReasoning.from_response(response_json).split() message = litellm.Message(content=content, reasoning_content=reasoning_content) model_response.choices[0].message = message model_response.choices[0].finish_reason = "stop" else: - response_text = response_json.get("response", "") - content = None - reasoning_content = None - if response_text is not None and isinstance(response_text, str): - reasoning_content, content = _parse_content_for_reasoning(response_text) - else: - content = response_text + reasoning_content, content = _OllamaGenerateReasoning.from_response(response_json).split() model_response.choices[0].message.content = content model_response.choices[0].message.reasoning_content = reasoning_content model_response.created = int(time.time()) diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py index 28e86e40944..d6215a742f0 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py @@ -414,6 +414,163 @@ class TestOllamaConfig: ) assert result.choices[0]["finish_reason"] == "stop" + def _transform( + self, response_json: dict[str, object], request_data: dict[str, object] | None = None + ) -> ModelResponse: + config = OllamaConfig() + + raw_response = MagicMock() + raw_response.json.return_value = response_json + + mock_encoding = MagicMock() + mock_encoding.encode.return_value = [1, 2, 3] + + return config.transform_response( + model="gpt-oss:120b", + raw_response=raw_response, + model_response=ModelResponse( + id="test_id", + choices=[{"message": Message(content="")}], + ), + logging_obj=MagicMock(), + request_data=request_data or {}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=mock_encoding, + ) + + def test_transform_response_with_thinking_field(self): + """`/api/generate` returns reasoning in a top-level `thinking` field, which must + reach `reasoning_content` instead of being dropped.""" + result = self._transform( + { + "response": "OK", + "thinking": 'We need to reply with exactly "OK".', + "prompt_eval_count": 15, + "eval_count": 8, + } + ) + + assert result.choices[0]["message"].reasoning_content == 'We need to reply with exactly "OK".' + assert result.choices[0]["message"].content == "OK" + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_with_thinking_field_and_empty_response(self): + """A model that spends its whole turn reasoning leaves `response` empty; the + reasoning still has to be surfaced rather than billed and discarded.""" + result = self._transform( + { + "response": "", + "thinking": "Entire turn went into reasoning.", + "eval_count": 96, + } + ) + + assert result.choices[0]["message"].reasoning_content == "Entire turn went into reasoning." + assert result.choices[0]["message"].content == "" + + def test_transform_response_thinking_field_wins_over_inline_tags(self): + """When both shapes are present the field wins, matching the `ollama_chat` transport.""" + result = self._transform( + { + "response": "inlineAnswer", + "thinking": "from field", + } + ) + + assert result.choices[0]["message"].reasoning_content == "from field" + assert result.choices[0]["message"].content == "inlineAnswer" + + def test_transform_response_json_mode_non_json_text_with_thinking_field(self): + """JSON mode falls back to text handling when the payload is not JSON, so the + `thinking` field has to be picked up on that path too.""" + result = self._transform( + { + "response": "not valid json", + "thinking": "reasoning in json mode", + }, + request_data={"format": "json"}, + ) + + assert result.choices[0]["message"].reasoning_content == "reasoning in json mode" + assert result.choices[0]["message"].content == "not valid json" + + def test_transform_response_empty_thinking_field_falls_back_to_tags(self): + """An empty `thinking` field must not mask inline `` tags.""" + result = self._transform( + { + "response": "inline reasoningAnswer", + "thinking": "", + } + ) + + assert result.choices[0]["message"].reasoning_content == "inline reasoning" + assert result.choices[0]["message"].content == "Answer" + + def test_transform_response_json_mode_valid_json_keeps_thinking_field(self): + """A valid JSON `response` is returned as content, and the reasoning that came + with it must not be dropped.""" + result = self._transform( + { + "response": '{"answer": 42}', + "thinking": "reasoned before answering in json", + }, + request_data={"format": "json"}, + ) + + assert result.choices[0]["message"].content == '{"answer": 42}' + assert result.choices[0]["message"].reasoning_content == "reasoned before answering in json" + assert result.choices[0]["finish_reason"] == "stop" + + def test_transform_response_json_mode_function_call_keeps_thinking_field(self): + """A JSON `response` shaped like a function call becomes a tool call, and the + reasoning behind the call must survive alongside it.""" + result = self._transform( + { + "response": '{"name": "get_weather", "arguments": {"city": "Paris"}}', + "thinking": "the user wants weather, so call the tool", + }, + request_data={"format": "json"}, + ) + + message = result.choices[0]["message"] + assert message.tool_calls is not None + assert message.tool_calls[0].function.name == "get_weather" + assert message.reasoning_content == "the user wants weather, so call the tool" + assert result.choices[0]["finish_reason"] == "tool_calls" + + def test_transform_response_json_mode_empty_response_keeps_thinking_field(self): + """In JSON mode a model that spends its whole turn reasoning leaves `response` + empty; the reasoning must still come back instead of a blank message.""" + result = self._transform( + { + "response": "", + "thinking": "all of the tokens went into reasoning", + }, + request_data={"format": "json"}, + ) + + assert result.choices[0]["message"].content == "" + assert result.choices[0]["message"].reasoning_content == "all of the tokens went into reasoning" + + def test_transform_response_null_response_keeps_content_null(self): + """Ollama sends `response: null` when the reply carries no text; that stays null + rather than becoming an empty string, while `thinking` is still surfaced.""" + result = self._transform({"response": None, "thinking": "reasoning only"}) + + assert result.choices[0]["message"].content is None + assert result.choices[0]["message"].reasoning_content == "reasoning only" + + def test_transform_response_malformed_reasoning_fields_do_not_crash(self): + """A reply whose `response` and `thinking` are not strings must still produce a + response instead of raising, with no reasoning invented.""" + result = self._transform({"response": 5, "thinking": ["not", "a", "string"]}) + + assert result.choices[0]["message"].reasoning_content is None + assert result.choices[0]["message"].content == "" + assert result.choices[0]["finish_reason"] == "stop" + class TestOllamaTextCompletionResponseIterator: def test_chunk_parser_with_thinking_field(self):