From bf438da4810b3c729216281a57e6789dc3578b3b Mon Sep 17 00:00:00 2001 From: Meryem Sakin Date: Thu, 10 Sep 2026 20:53:57 +0300 Subject: [PATCH 1/7] fix(ollama): use native tool calling for ollama/ models Requests with tools on the ollama/ prefix went to /api/generate with the tools pasted into a "Produce JSON OUTPUT ONLY" system prompt and format=json. On the turn after a tool result the model is still forced to emit a function call, so agents loop forever (#40575), and streamed calls come back as plain text (#35711) Route them through the ollama_chat /api/chat path instead, which sends tools and tool messages natively. This is the same change #18924 made for ollama_chat --- litellm/main.py | 2 + .../test_ollama_completion_transformation.py | 104 +++++++++++++++++- .../test_system_message_format_bug.py | 5 +- 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 4f1c57adb53..90a9961a384 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5308,6 +5308,8 @@ def completion( GenericLiteLLMParams(**_supplemental_provider_params) if _supplemental_provider_params else None ), ) + if custom_llm_provider == "ollama" and (tools is not None or functions is not None): + custom_llm_provider = "ollama_chat" # rebind-ok: /api/generate has no native tool calling ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py index b2071155f3f..151bd66e98f 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py @@ -6,7 +6,7 @@ import httpx import pytest import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.ollama.completion.transformation import ( OllamaConfig, OllamaTextCompletionResponseIterator, @@ -544,3 +544,105 @@ async def test_ollama_async_completion_inlines_remote_images_off_the_event_loop( assert response.choices[0].message.content == "Green" assert async_only_image_fetch.fetched == [image_url] assert captured["body"]["images"] == [async_only_image_fetch.base64_png] + + +GRAPH_STATS_TOOLS = [ + { + "type": "function", + "function": { + "name": "graph_stats", + "description": "Return node and edge counts of the code graph", + "parameters": {"type": "object", "properties": {}}, + }, + } +] + + +def test_ollama_tool_result_turn_is_sent_to_native_chat_api(): + """https://github.com/BerriAI/litellm/issues/40575""" + requests = [] + + def handle(request): + requests.append((request.url.path, json.loads(request.content))) + return httpx.Response( + 200, + json={ + "model": "qwen3.8:27b", + "message": {"role": "assistant", "content": "The graph has 190921 nodes."}, + "done": True, + "done_reason": "stop", + "prompt_eval_count": 1, + "eval_count": 1, + }, + ) + + response = litellm.completion( + model="ollama/qwen3.8:27b", + messages=[ + {"role": "user", "content": "How many nodes does the graph have?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "graph_stats", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "name": "graph_stats", "content": '{"nodes": 190921}'}, + ], + tools=GRAPH_STATS_TOOLS, + api_base="http://ollama.example:11434", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))), + ) + + assert [path for path, _ in requests] == ["/api/chat"] + body = requests[0][1] + assert body["tools"] == GRAPH_STATS_TOOLS + assert "format" not in body + assert [m["role"] for m in body["messages"]] == ["user", "assistant", "tool"] + assert body["messages"][2]["content"] == '{"nodes": 190921}' + assert response.choices[0].message.content == "The graph has 190921 nodes." + assert response.choices[0].message.tool_calls is None + assert response.choices[0].finish_reason == "stop" + + +def test_ollama_streamed_tool_call_is_returned_as_tool_call(): + """https://github.com/BerriAI/litellm/issues/35711""" + chunks = [ + { + "model": "qwen3.8:27b", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [{"function": {"name": "graph_stats", "arguments": {}}}], + }, + "done": False, + }, + { + "model": "qwen3.8:27b", + "message": {"role": "assistant", "content": ""}, + "done": True, + "done_reason": "stop", + "prompt_eval_count": 1, + "eval_count": 1, + }, + ] + + def handle(request): + assert request.url.path == "/api/chat" + return httpx.Response(200, content="\n".join(json.dumps(chunk) for chunk in chunks).encode()) + + streamed = list( + litellm.completion( + model="ollama/qwen3.8:27b", + messages=[{"role": "user", "content": "How many nodes does the graph have?"}], + tools=GRAPH_STATS_TOOLS, + stream=True, + api_base="http://ollama.example:11434", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))), + ) + ) + + tool_calls = [tool_call for chunk in streamed for tool_call in chunk.choices[0].delta.tool_calls or []] + assert [tool_call.function.name for tool_call in tool_calls] == ["graph_stats"] + assert "".join(chunk.choices[0].delta.content or "" for chunk in streamed) == "" + assert streamed[-1].choices[0].finish_reason == "tool_calls" diff --git a/tests/test_litellm/test_system_message_format_bug.py b/tests/test_litellm/test_system_message_format_bug.py index 375c4ea22d4..6d2a079f8b0 100644 --- a/tests/test_litellm/test_system_message_format_bug.py +++ b/tests/test_litellm/test_system_message_format_bug.py @@ -2,6 +2,7 @@ Test for GitHub issue #11267 - System message format issue with Ollama + tools """ +import copy from unittest.mock import patch @@ -49,6 +50,8 @@ def test_system_message_format_issue_reproduction(): } ] + original_messages = copy.deepcopy(messages) + response = completion( model=model, messages=messages, @@ -57,7 +60,7 @@ def test_system_message_format_issue_reproduction(): mock_response=True, ) - assert len(messages[1]["content"]) == 2 + assert messages == original_messages if __name__ == "__main__": From 4f7f73b57ef07332ca83dc31342a9537f42878d8 Mon Sep 17 00:00:00 2001 From: Meryem Sakin Date: Thu, 10 Sep 2026 20:55:59 +0300 Subject: [PATCH 2/7] refactor(ollama): remove the JSON prompt tool-calling emulation With ollama/ tool requests going through /api/chat, nothing reaches the emulation anymore. The removed block in get_optional_params only ever ran for ollama (the long != chain was dead after its first == check), and it also flipped the global litellm.add_function_to_prompt on the first tool request. function_call_prompt mutated the caller's system message in place, so a reused message list picked up another copy of the prompt on every turn --- .../prompt_templates/factory.py | 21 -------- litellm/main.py | 7 --- litellm/utils.py | 48 ------------------- 3 files changed, 76 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index ece619e3883..e303a18962f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -5130,27 +5130,6 @@ def _bedrock_tools_pt(tools: list, model: str | None = None) -> list[BedrockTool return tool_block_list -# Function call template -def function_call_prompt(messages: list, functions: list): - function_prompt = """Produce JSON OUTPUT ONLY! Adhere to this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}} The following functions are available to you:""" - for function in functions: - function_prompt += f"""\n{function}\n""" - - function_added_to_prompt = False - for message in messages: - if "system" in message["role"]: - if isinstance(message["content"], str): - message["content"] += f""" {function_prompt}""" - else: - message["content"].append({"type": "text", "text": f""" {function_prompt}"""}) - function_added_to_prompt = True - - if function_added_to_prompt is False: - messages.append({"role": "system", "content": f"""{function_prompt}"""}) - - return messages - - def response_schema_prompt(model: str, response_schema: dict) -> str: """ Decides if a user-defined custom prompt or default needs to be used diff --git a/litellm/main.py b/litellm/main.py index 90a9961a384..d69b737ce04 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -180,7 +180,6 @@ from .litellm_core_utils.prompt_templates.common_utils import ( ) from .litellm_core_utils.prompt_templates.factory import ( custom_prompt, - function_call_prompt, map_system_message_pt, ollama_pt, prompt_factory, @@ -5467,12 +5466,6 @@ def completion( provider_config=provider_config, ) - if litellm.add_function_to_prompt and optional_params.get( - "functions_unsupported_model", None - ): # if user opts to add it to prompt, when API doesn't support function calling - functions_unsupported_model: Final = optional_params.pop("functions_unsupported_model") - messages = function_call_prompt(messages=messages, functions=functions_unsupported_model) - # For logging - save the values of the litellm-specific params passed in litellm_params = get_litellm_params( acompletion=acompletion, diff --git a/litellm/utils.py b/litellm/utils.py index 917af2b89d4..f13d2c80964 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4098,54 +4098,6 @@ def pre_process_optional_params(passed_params: dict, non_default_params: dict, c non_default_params=passed_params, optional_params=optional_params ) - ## raise exception if function calling passed in for a provider that doesn't support it - if "functions" in non_default_params or "function_call" in non_default_params or "tools" in non_default_params: - if ( - custom_llm_provider == "ollama" - and custom_llm_provider != "text-completion-openai" - and custom_llm_provider != "azure" - and custom_llm_provider != "vertex_ai" - and custom_llm_provider != "anyscale" - and custom_llm_provider != "together_ai" - and custom_llm_provider != "groq" - and custom_llm_provider != "nvidia_nim" - and custom_llm_provider != "cerebras" - and custom_llm_provider != "xai" - and custom_llm_provider != "ai21_chat" - and custom_llm_provider != "volcengine" - and custom_llm_provider != "deepseek" - and custom_llm_provider != "codestral" - and custom_llm_provider != "mistral" - and custom_llm_provider != "anthropic" - and custom_llm_provider != "cohere_chat" - and custom_llm_provider != "cohere" - and custom_llm_provider != "bedrock" - and custom_llm_provider != "ollama_chat" - and custom_llm_provider != "openrouter" - and custom_llm_provider != "vercel_ai_gateway" - and custom_llm_provider != "nebius" - and custom_llm_provider != "wandb" - and custom_llm_provider not in litellm.openai_compatible_providers - ): - if custom_llm_provider == "ollama": - # ollama actually supports json output - optional_params["format"] = "json" - litellm.add_function_to_prompt = True # so that main.py adds the function call to the prompt - if "tools" in non_default_params: - optional_params["functions_unsupported_model"] = non_default_params.pop("tools") - non_default_params.pop("tool_choice", None) # causes ollama requests to hang - elif "functions" in non_default_params: - optional_params["functions_unsupported_model"] = non_default_params.pop("functions") - elif litellm.add_function_to_prompt: # if user opts to add it to prompt instead - optional_params["functions_unsupported_model"] = non_default_params.pop( - "tools", non_default_params.pop("functions", None) - ) - else: - raise UnsupportedParamsError( - status_code=500, - message=f"Function calling is not supported by {custom_llm_provider}.", - ) - return optional_params From 3f69357104c67e15cb13a2c45fc609d51eae7f81 Mon Sep 17 00:00:00 2001 From: Meryem Sakin Date: Thu, 10 Sep 2026 21:55:41 +0300 Subject: [PATCH 3/7] fix(ollama): preserve compatibility when routing native tools --- litellm/llms/ollama/chat/transformation.py | 5 +- litellm/main.py | 9 +- .../test_ollama_completion_transformation.py | 89 ++++++++++++++++++- 3 files changed, 97 insertions(+), 6 deletions(-) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 181894646e3..55d6603f8ab 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -24,6 +24,7 @@ from litellm.types.llms.ollama import ( from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantToolCall, + ChatCompletionToolParam, ChatCompletionUsageBlock, ) from litellm.types.utils import ModelResponse, ModelResponseStream @@ -184,7 +185,9 @@ class OllamaChatConfig(BaseConfig): optional_params["tools"] = value if param == "functions": - optional_params["tools"] = value + optional_params["tools"] = tuple( + ChatCompletionToolParam(type="function", function=function) for function in value + ) non_default_params.pop("tool_choice", None) # causes ollama requests to hang non_default_params.pop("functions", None) # causes ollama requests to hang return optional_params diff --git a/litellm/main.py b/litellm/main.py index d69b737ce04..9b81cec1c7e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5307,8 +5307,15 @@ def completion( GenericLiteLLMParams(**_supplemental_provider_params) if _supplemental_provider_params else None ), ) - if custom_llm_provider == "ollama" and (tools is not None or functions is not None): + if custom_llm_provider == "ollama" and (tools or functions): custom_llm_provider = "ollama_chat" # rebind-ok: /api/generate has no native tool calling + if api_base is not None: + api_base = api_base.rstrip("/").removesuffix( + "/api/generate" + ) # rebind-ok: preserve generate URLs for native tool requests + elif custom_llm_provider == "ollama": + tools = None # rebind-ok: empty tools must not change plain completion behavior + functions = None # rebind-ok: empty functions must not change plain completion behavior ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py index 151bd66e98f..9942f44dcca 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py @@ -1,11 +1,12 @@ import json -from litellm._uuid import uuid +from typing import Final from unittest.mock import MagicMock, patch import httpx import pytest import litellm +from litellm._uuid import uuid from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.ollama.completion.transformation import ( OllamaConfig, @@ -476,7 +477,7 @@ class TestOllamaTextCompletionResponseIterator: # Updated to handle ModelResponseStream return type assert isinstance(result, ModelResponseStream) assert result.choices and result.choices[0].delta is not None - assert result.choices[0].delta.content == None + assert result.choices[0].delta.content is None assert getattr(result.choices[0].delta, "reasoning_content", None) == "" def test_chunk_parser_done_chunk(self): @@ -558,7 +559,17 @@ GRAPH_STATS_TOOLS = [ ] -def test_ollama_tool_result_turn_is_sent_to_native_chat_api(): +@pytest.mark.parametrize( + "api_base", + [ + "http://ollama.example:11434", + "http://ollama.example:11434/", + "http://ollama.example:11434/api/generate", + "http://ollama.example:11434/api/generate/", + "http://ollama.example:11434/api/chat", + ], +) +def test_ollama_tool_result_turn_is_sent_to_native_chat_api(api_base: str): """https://github.com/BerriAI/litellm/issues/40575""" requests = [] @@ -590,7 +601,7 @@ def test_ollama_tool_result_turn_is_sent_to_native_chat_api(): {"role": "tool", "tool_call_id": "call_1", "name": "graph_stats", "content": '{"nodes": 190921}'}, ], tools=GRAPH_STATS_TOOLS, - api_base="http://ollama.example:11434", + api_base=api_base, client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))), ) @@ -646,3 +657,73 @@ def test_ollama_streamed_tool_call_is_returned_as_tool_call(): assert [tool_call.function.name for tool_call in tool_calls] == ["graph_stats"] assert "".join(chunk.choices[0].delta.content or "" for chunk in streamed) == "" assert streamed[-1].choices[0].finish_reason == "tool_calls" + + +@pytest.mark.parametrize("empty_parameter", ["none", "tools", "functions"]) +def test_ollama_empty_tools_preserve_generate_request(empty_parameter: str) -> None: + def handle(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + assert request.url.path == "/api/generate" + assert "format" not in body + assert "tools" not in body + return httpx.Response(200, json={"response": "Hello", "done": True}) + + response: Final = litellm.completion( + model="ollama/qwen3.8:27b", + messages=[{"role": "user", "content": "Hello"}], + tools=[] if empty_parameter == "tools" else None, + functions=[] if empty_parameter == "functions" else None, + api_base="http://ollama.example:11434/api/generate", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))), + ) + assert response.choices[0].message.content == "Hello" + + +def test_ollama_native_tool_support_error_is_preserved() -> None: + def handle(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/chat" + return httpx.Response(400, json={"error": "model does not support tools"}) + + with pytest.raises(litellm.BadRequestError, match="does not support tools"): + litellm.completion( + model="ollama/qwen3.8:27b", + messages=[{"role": "user", "content": "Hello"}], + tools=GRAPH_STATS_TOOLS, + api_base="http://ollama.example:11434/api/generate", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))), + num_retries=0, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("legacy_functions", [False, True]) +async def test_ollama_async_native_tools(legacy_functions: bool) -> None: + def handle(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + assert request.url.path == "/prefix/api/chat" + assert body["tools"] == GRAPH_STATS_TOOLS + return httpx.Response( + 200, + json={ + "model": "qwen3.8:27b", + "message": {"role": "assistant", "content": "Hello"}, + "done": True, + "done_reason": "stop", + "prompt_eval_count": 1, + "eval_count": 1, + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as client: + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + handler.client = client + response: Final = await litellm.acompletion( + model="ollama/qwen3.8:27b", + messages=[{"role": "user", "content": "Hello"}], + tools=None if legacy_functions else GRAPH_STATS_TOOLS, + functions=[GRAPH_STATS_TOOLS[0]["function"]] if legacy_functions else None, + api_base="http://ollama.example:11434/prefix/api/generate/", + client=handler, + ) + assert response.choices[0].message.content == "Hello" From fc9ee293e23d1b22b467aa4c57eecd99bff48835 Mon Sep 17 00:00:00 2001 From: Meryem Sakin Date: Thu, 10 Sep 2026 22:00:50 +0300 Subject: [PATCH 4/7] fix(ollama): keep routing suppression on assignment line --- litellm/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 9b81cec1c7e..5bcfeaebce9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5310,9 +5310,7 @@ def completion( if custom_llm_provider == "ollama" and (tools or functions): custom_llm_provider = "ollama_chat" # rebind-ok: /api/generate has no native tool calling if api_base is not None: - api_base = api_base.rstrip("/").removesuffix( - "/api/generate" - ) # rebind-ok: preserve generate URLs for native tool requests + api_base = api_base.rstrip("/").removesuffix("/api/generate") # rebind-ok: chat path elif custom_llm_provider == "ollama": tools = None # rebind-ok: empty tools must not change plain completion behavior functions = None # rebind-ok: empty functions must not change plain completion behavior From a0a9c05a5aca6cb1f471cba35c5d121e28d32c03 Mon Sep 17 00:00:00 2001 From: Meryem Sakin Date: Thu, 10 Sep 2026 22:36:33 +0300 Subject: [PATCH 5/7] fix(ollama): build the chat url from any api_base form The /api/generate suffix was only stripped from the api_base argument, so a global litellm.api_base ending in /api/generate still produced /api/generate/api/chat for rerouted tool requests. Normalize in OllamaChatConfig.get_complete_url instead, which every api_base source goes through, and drop the extra rebind in completion(). Trailing slashes no longer produce //api/chat either --- litellm/llms/ollama/chat/transformation.py | 10 ++-------- litellm/main.py | 2 -- .../ollama/test_ollama_chat_transformation.py | 20 +++++++++++++++++++ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 55d6603f8ab..bd624bfbfa0 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -222,14 +222,8 @@ class OllamaChatConfig(BaseConfig): Some providers need `model` in `api_base` """ - if api_base is None: - api_base = "http://localhost:11434" - if api_base.endswith("/api/chat"): - url = api_base - else: - url = f"{api_base}/api/chat" - - return url + base: Final = (api_base or "http://localhost:11434").rstrip("/").removesuffix("/api/generate") + return base if base.endswith("/api/chat") else f"{base}/api/chat" def transform_request( self, diff --git a/litellm/main.py b/litellm/main.py index 5bcfeaebce9..8984f6abfec 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5309,8 +5309,6 @@ def completion( ) if custom_llm_provider == "ollama" and (tools or functions): custom_llm_provider = "ollama_chat" # rebind-ok: /api/generate has no native tool calling - if api_base is not None: - api_base = api_base.rstrip("/").removesuffix("/api/generate") # rebind-ok: chat path elif custom_llm_provider == "ollama": tools = None # rebind-ok: empty tools must not change plain completion behavior functions = None # rebind-ok: empty functions must not change plain completion behavior diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index 25f9645faa0..69d6eb35ced 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -944,3 +944,23 @@ class TestOllamaToolCallTransformation: assert tool_msg["content"] == "Sunny, 72°F" assert "tool_call_id" in tool_msg, "tool_call_id must be forwarded to Ollama" assert tool_msg["tool_call_id"] == "call_abc123" + + +@pytest.mark.parametrize( + ("api_base", "expected_url"), + [ + (None, "http://localhost:11434/api/chat"), + ("http://ollama.example:11434", "http://ollama.example:11434/api/chat"), + ("http://ollama.example:11434/", "http://ollama.example:11434/api/chat"), + ("http://ollama.example:11434/api/chat", "http://ollama.example:11434/api/chat"), + ("http://ollama.example:11434/api/chat/", "http://ollama.example:11434/api/chat"), + ("http://ollama.example:11434/api/generate", "http://ollama.example:11434/api/chat"), + ("http://ollama.example:11434/prefix/api/generate/", "http://ollama.example:11434/prefix/api/chat"), + ], +) +def test_get_complete_url_points_at_chat_endpoint(api_base, expected_url): + url = OllamaChatConfig().get_complete_url( + api_base=api_base, api_key=None, model="qwen3.8:27b", optional_params={}, litellm_params={} + ) + + assert url == expected_url From b55377352edb4b5c5a23128c2f64f15407330869 Mon Sep 17 00:00:00 2001 From: Meryem Sakin Date: Thu, 10 Sep 2026 23:38:08 +0300 Subject: [PATCH 6/7] fix(ollama): keep the JSON prompt tool emulation behind add_function_to_prompt Review feedback: dropping the emulation was a backwards-incompatible change with no user-controlled flag, and the routing lived outside llms/ ollama/ tool requests still go to /api/chat by default. Setting the existing litellm.add_function_to_prompt flag (or --add_function_to_prompt on the proxy) keeps the old /api/generate JSON prompt path, now implemented inside OllamaConfig. The routing decision moved to llms/ollama/common_utils.py, and function_call_prompt returns new messages instead of editing the caller's list Also fixes the basedpyright reportOptionalIterable error on legacy functions --- .../prompt_templates/factory.py | 21 +++++++++++ litellm/llms/ollama/chat/transformation.py | 7 ++-- litellm/llms/ollama/common_utils.py | 12 +++++++ .../llms/ollama/completion/transformation.py | 19 ++++++++-- litellm/main.py | 11 +++--- ...llm_core_utils_prompt_templates_factory.py | 35 +++++++++++++++++++ .../test_ollama_completion_transformation.py | 34 ++++++++++++++++++ 7 files changed, 126 insertions(+), 13 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index e303a18962f..ba9b04ab222 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -32,6 +32,7 @@ from litellm.types.llms.openai import ( ChatCompletionFileObject, ChatCompletionFunctionMessage, ChatCompletionImageObject, + ChatCompletionSystemMessage, ChatCompletionTextObject, ChatCompletionToolCallFunctionChunk, ChatCompletionToolMessage, @@ -5130,6 +5131,26 @@ def _bedrock_tools_pt(tools: list, model: str | None = None) -> list[BedrockTool return tool_block_list +def _append_function_prompt(message: ChatCompletionSystemMessage, text: str) -> ChatCompletionSystemMessage: + content: Final = message["content"] + if isinstance(content, str): + return {**message, "content": content + text} + return {**message, "content": [*content, ChatCompletionTextObject(type="text", text=text)]} + + +def function_call_prompt(messages: Sequence[AllMessageValues], function_descriptions: str) -> list[AllMessageValues]: + function_prompt: Final = ( + 'Produce JSON OUTPUT ONLY! Adhere to this format {"name": "function_name", "arguments":{"argument_name": ' + '"argument_value"}} The following functions are available to you:' + function_descriptions + ) + if not any(message["role"] == "system" for message in messages): + return [*messages, ChatCompletionSystemMessage(role="system", content=function_prompt)] + return [ + _append_function_prompt(message, f" {function_prompt}") if message["role"] == "system" else message + for message in messages + ] + + def response_schema_prompt(model: str, response_schema: dict) -> str: """ Decides if a user-defined custom prompt or default needs to be used diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index bd624bfbfa0..db97c139e02 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -24,7 +24,6 @@ from litellm.types.llms.ollama import ( from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantToolCall, - ChatCompletionToolParam, ChatCompletionUsageBlock, ) from litellm.types.utils import ModelResponse, ModelResponseStream @@ -184,10 +183,8 @@ class OllamaChatConfig(BaseConfig): if param == "tools": optional_params["tools"] = value - if param == "functions": - optional_params["tools"] = tuple( - ChatCompletionToolParam(type="function", function=function) for function in value - ) + if param == "functions" and value: + optional_params["tools"] = [{"type": "function", "function": function} for function in value] non_default_params.pop("tool_choice", None) # causes ollama requests to hang non_default_params.pop("functions", None) # causes ollama requests to hang return optional_params diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index ed4bab22a84..0111340ca8e 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -11,6 +11,18 @@ class OllamaError(BaseLLMException): super().__init__(status_code=status_code, message=message, headers=headers) +def resolve_ollama_tool_calling_provider( + custom_llm_provider: str, has_tools: bool, add_function_to_prompt: bool +) -> str: + """ + /api/generate has no native tool calling, so ollama/ tool requests go through the ollama_chat + adapter unless add_function_to_prompt opts back into the legacy JSON prompt emulation + """ + if custom_llm_provider == "ollama" and has_tools and not add_function_to_prompt: + return "ollama_chat" + return custom_llm_provider + + def _convert_image(image): """ Convert image to base64 encoded image if not already in base64 format diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index a1340ba1952..7a748fb260f 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_ollama_image, custom_prompt, + function_call_prompt, ollama_pt, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( @@ -159,6 +160,9 @@ class OllamaConfig(BaseConfig): "response_format", "max_completion_tokens", "reasoning_effort", + "tools", + "tool_choice", + "functions", ] def map_openai_params( @@ -193,6 +197,9 @@ class OllamaConfig(BaseConfig): optional_params["format"] = "json" elif value["type"] == "json_schema": optional_params["format"] = value["json_schema"]["schema"] + elif param in ("tools", "functions") and value: + optional_params["format"] = "json" + optional_params["prompted_functions"] = "".join(f"\n{function}\n" for function in value) return optional_params @@ -377,6 +384,12 @@ class OllamaConfig(BaseConfig): headers: dict, ) -> dict: custom_prompt_dict: Final = litellm_params.get("custom_prompt_dict") or litellm.custom_prompt_dict + prompted_functions: Final = optional_params.pop("prompted_functions", None) + prompt_messages: Final = ( + function_call_prompt(messages=messages, function_descriptions=prompted_functions) + if isinstance(prompted_functions, str) + else messages + ) text_completion_request: Final = litellm_params.get("text_completion") if model in custom_prompt_dict: @@ -386,12 +399,12 @@ class OllamaConfig(BaseConfig): role_dict=model_prompt_details["roles"], initial_prompt_value=model_prompt_details["initial_prompt_value"], final_prompt_value=model_prompt_details["final_prompt_value"], - messages=messages, + messages=prompt_messages, ) elif text_completion_request: # handle `/completions` requests - ollama_prompt = get_str_from_messages(messages=messages) + ollama_prompt = get_str_from_messages(messages=prompt_messages) else: # handle `/chat/completions` requests - modified_prompt: Final = ollama_pt(model=model, messages=messages) + modified_prompt: Final = ollama_pt(model=model, messages=prompt_messages) if isinstance(modified_prompt, dict): ollama_prompt, images = ( modified_prompt["prompt"], diff --git a/litellm/main.py b/litellm/main.py index 8984f6abfec..6db13ddcdb7 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -220,6 +220,7 @@ from .llms.nvidia_riva.audio_transcription.transformation import ( NvidiaRivaAudioTranscriptionConfig, ) from .llms.oci.chat.transformation import OCIChatConfig +from .llms.ollama.common_utils import resolve_ollama_tool_calling_provider from .llms.ollama.completion import handler as ollama from .llms.oobabooga.chat import oobabooga from .llms.openai.completion.handler import OpenAITextCompletion @@ -5307,11 +5308,11 @@ def completion( GenericLiteLLMParams(**_supplemental_provider_params) if _supplemental_provider_params else None ), ) - if custom_llm_provider == "ollama" and (tools or functions): - custom_llm_provider = "ollama_chat" # rebind-ok: /api/generate has no native tool calling - elif custom_llm_provider == "ollama": - tools = None # rebind-ok: empty tools must not change plain completion behavior - functions = None # rebind-ok: empty functions must not change plain completion behavior + custom_llm_provider = resolve_ollama_tool_calling_provider( # rebind-ok: ollama tools use the chat adapter + custom_llm_provider, + has_tools=True if tools or functions else False, + add_function_to_prompt=litellm.add_function_to_prompt, + ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 66d10fd1407..d6bac6b45f6 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( _convert_to_bedrock_tool_call_result, anthropic_messages_pt, convert_to_gemini_tool_call_result, + function_call_prompt, make_valid_bedrock_tool_name, ollama_pt, sanitize_messages_for_tool_calling, @@ -3721,3 +3722,37 @@ def test_convert_to_anthropic_tool_invoke_keeps_paired_server_tool_use(): }, server_result, ] + + +FUNCTION_PROMPT_DESCRIPTIONS: Final = "\n{'name': 'graph_stats'}\n" + + +@pytest.mark.parametrize( + ("messages", "expected_system_contents"), + [ + ([{"role": "user", "content": "hi"}], None), + ([{"role": "system", "content": "Be brief."}, {"role": "user", "content": "hi"}], "Be brief. "), + ( + [{"role": "system", "content": [{"type": "text", "text": "Be brief."}]}, {"role": "user", "content": "hi"}], + [{"type": "text", "text": "Be brief."}], + ), + ], +) +def test_function_call_prompt_returns_new_messages(messages, expected_system_contents): + original: Final = json.loads(json.dumps(messages)) + + result: Final = function_call_prompt(messages=messages, function_descriptions=FUNCTION_PROMPT_DESCRIPTIONS) + + assert messages == original + system_messages: Final = [m for m in result if m["role"] == "system"] + assert len(system_messages) == 1 + content: Final = system_messages[0]["content"] + prompt_text: Final = content if isinstance(content, str) else content[-1]["text"] + assert "Produce JSON OUTPUT ONLY" in prompt_text + assert "graph_stats" in prompt_text + if expected_system_contents is None: + assert result[:-1] == original + elif isinstance(expected_system_contents, str): + assert content.startswith(expected_system_contents) + else: + assert content[:-1] == expected_system_contents diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py index 9942f44dcca..509b1e80e72 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py @@ -727,3 +727,37 @@ async def test_ollama_async_native_tools(legacy_functions: bool) -> None: client=handler, ) assert response.choices[0].message.content == "Hello" + + +def test_ollama_add_function_to_prompt_keeps_legacy_json_emulation(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "add_function_to_prompt", True) + requests = [] + + def handle(request: httpx.Request) -> httpx.Response: + requests.append((request.url.path, json.loads(request.content))) + return httpx.Response( + 200, json={"response": '{"name": "graph_stats", "arguments": {}}', "done": True, "prompt_eval_count": 1} + ) + + messages: Final = [ + {"role": "system", "content": "You are a graph assistant."}, + {"role": "user", "content": "How many nodes does the graph have?"}, + ] + + response: Final = litellm.completion( + model="ollama/qwen3.8:27b", + messages=messages, + tools=GRAPH_STATS_TOOLS, + tool_choice="auto", + api_base="http://ollama.example:11434", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))), + ) + + assert [path for path, _ in requests] == ["/api/generate"] + body: Final = requests[0][1] + assert body["format"] == "json" + assert "Produce JSON OUTPUT ONLY" in body["prompt"] + assert "graph_stats" in body["prompt"] + assert "prompted_functions" not in body["options"] + assert response.choices[0].message.tool_calls[0].function.name == "graph_stats" + assert response.choices[0].finish_reason == "tool_calls" From 5476e05bbe50f20fc63cee8108c3d3cc190d64ad Mon Sep 17 00:00:00 2001 From: Meryem Sakin Date: Fri, 11 Sep 2026 00:11:18 +0300 Subject: [PATCH 7/7] fix(ollama): check for tools before resolving the ollama route The ternary that turned tools into a bool tripped the SIM210 strict-rule budget. A plain if in completion() avoids both that and passing the untyped tool lists into a typed helper --- litellm/llms/ollama/common_utils.py | 10 ++++------ litellm/main.py | 9 ++++----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 0111340ca8e..36fe39982a5 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -11,14 +11,12 @@ class OllamaError(BaseLLMException): super().__init__(status_code=status_code, message=message, headers=headers) -def resolve_ollama_tool_calling_provider( - custom_llm_provider: str, has_tools: bool, add_function_to_prompt: bool -) -> str: +def resolve_ollama_tool_calling_provider(custom_llm_provider: str, add_function_to_prompt: bool) -> str: """ - /api/generate has no native tool calling, so ollama/ tool requests go through the ollama_chat - adapter unless add_function_to_prompt opts back into the legacy JSON prompt emulation + For requests with tools: /api/generate has no native tool calling, so ollama/ goes through the + ollama_chat adapter unless add_function_to_prompt opts back into the legacy JSON prompt emulation """ - if custom_llm_provider == "ollama" and has_tools and not add_function_to_prompt: + if custom_llm_provider == "ollama" and not add_function_to_prompt: return "ollama_chat" return custom_llm_provider diff --git a/litellm/main.py b/litellm/main.py index 6db13ddcdb7..18e397a1f6d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5308,11 +5308,10 @@ def completion( GenericLiteLLMParams(**_supplemental_provider_params) if _supplemental_provider_params else None ), ) - custom_llm_provider = resolve_ollama_tool_calling_provider( # rebind-ok: ollama tools use the chat adapter - custom_llm_provider, - has_tools=True if tools or functions else False, - add_function_to_prompt=litellm.add_function_to_prompt, - ) + if tools or functions: + custom_llm_provider = resolve_ollama_tool_calling_provider( # rebind-ok: ollama tools use the chat adapter + custom_llm_provider, add_function_to_prompt=litellm.add_function_to_prompt + ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check(