scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 156

Merge 5476e05bbe into 0c98afa780

Browse source
This commit is contained in:
Meryem Sakin 2026-09-12 20:04:40 +02:00 • committed by GitHub
commit 3024970d05
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 331 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -32,6 +32,7 @@ from litellm.types.llms.openai import (
ChatCompletionFileObject,
ChatCompletionFunctionMessage,
ChatCompletionImageObject,
ChatCompletionSystemMessage,
ChatCompletionTextObject,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolMessage,
@ -5130,25 +5131,24 @@ def _bedrock_tools_pt(tools: list, model: str | None = None) -> list[BedrockTool
return tool_block_list
# Function call template
def function_call_prompt(messages: list, functions: list):
function_prompt = """Produce JSON OUTPUT ONLY! Adhere to this format {"name": "function_name", "arguments":{"argument_name": "argument_value"}} The following functions are available to you:"""
for function in functions:
function_prompt += f"""\n{function}\n"""
def _append_function_prompt(message: ChatCompletionSystemMessage, text: str) -> ChatCompletionSystemMessage:
content: Final = message["content"]
if isinstance(content, str):
return {**message, "content": content + text}
return {**message, "content": [*content, ChatCompletionTextObject(type="text", text=text)]}
function_added_to_prompt = False
for message in messages:
if "system" in message["role"]:
if isinstance(message["content"], str):
message["content"] += f""" {function_prompt}"""
else:
message["content"].append({"type": "text", "text": f""" {function_prompt}"""})
function_added_to_prompt = True
if function_added_to_prompt is False:
messages.append({"role": "system", "content": f"""{function_prompt}"""})
return messages
def function_call_prompt(messages: Sequence[AllMessageValues], function_descriptions: str) -> list[AllMessageValues]:
function_prompt: Final = (
'Produce JSON OUTPUT ONLY! Adhere to this format {"name": "function_name", "arguments":{"argument_name": '
'"argument_value"}} The following functions are available to you:' + function_descriptions
)
if not any(message["role"] == "system" for message in messages):
return [*messages, ChatCompletionSystemMessage(role="system", content=function_prompt)]
return [
_append_function_prompt(message, f" {function_prompt}") if message["role"] == "system" else message
for message in messages
]
def response_schema_prompt(model: str, response_schema: dict) -> str:

14
litellm/llms/ollama/chat/transformation.py
View file

@ -183,8 +183,8 @@ class OllamaChatConfig(BaseConfig):
if param == "tools":
optional_params["tools"] = value
if param == "functions":
optional_params["tools"] = value
if param == "functions" and value:
optional_params["tools"] = [{"type": "function", "function": function} for function in value]
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
non_default_params.pop("functions", None) # causes ollama requests to hang
return optional_params
@ -219,14 +219,8 @@ class OllamaChatConfig(BaseConfig):
Some providers need `model` in `api_base`
"""
if api_base is None:
api_base = "http://localhost:11434"
if api_base.endswith("/api/chat"):
url = api_base
else:
url = f"{api_base}/api/chat"
return url
base: Final = (api_base or "http://localhost:11434").rstrip("/").removesuffix("/api/generate")
return base if base.endswith("/api/chat") else f"{base}/api/chat"
def transform_request(
self,

10
litellm/llms/ollama/common_utils.py
View file

@ -11,6 +11,16 @@ class OllamaError(BaseLLMException):
super().__init__(status_code=status_code, message=message, headers=headers)
def resolve_ollama_tool_calling_provider(custom_llm_provider: str, add_function_to_prompt: bool) -> str:
"""
For requests with tools: /api/generate has no native tool calling, so ollama/ goes through the
ollama_chat adapter unless add_function_to_prompt opts back into the legacy JSON prompt emulation
"""
if custom_llm_provider == "ollama" and not add_function_to_prompt:
return "ollama_chat"
return custom_llm_provider
def _convert_image(image):
"""
Convert image to base64 encoded image if not already in base64 format

19
litellm/llms/ollama/completion/transformation.py
View file

@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_ollama_image,
custom_prompt,
function_call_prompt,
ollama_pt,
)
from litellm.litellm_core_utils.prompt_templates.image_handling import (
@ -159,6 +160,9 @@ class OllamaConfig(BaseConfig):
"response_format",
"max_completion_tokens",
"reasoning_effort",
"tools",
"tool_choice",
"functions",
]
def map_openai_params(
@ -193,6 +197,9 @@ class OllamaConfig(BaseConfig):
optional_params["format"] = "json"
elif value["type"] == "json_schema":
optional_params["format"] = value["json_schema"]["schema"]
elif param in ("tools", "functions") and value:
optional_params["format"] = "json"
optional_params["prompted_functions"] = "".join(f"\n{function}\n" for function in value)
return optional_params
@ -377,6 +384,12 @@ class OllamaConfig(BaseConfig):
headers: dict,
) -> dict:
custom_prompt_dict: Final = litellm_params.get("custom_prompt_dict") or litellm.custom_prompt_dict
prompted_functions: Final = optional_params.pop("prompted_functions", None)
prompt_messages: Final = (
function_call_prompt(messages=messages, function_descriptions=prompted_functions)
if isinstance(prompted_functions, str)
else messages
)
text_completion_request: Final = litellm_params.get("text_completion")
if model in custom_prompt_dict:
@ -386,12 +399,12 @@ class OllamaConfig(BaseConfig):
role_dict=model_prompt_details["roles"],
initial_prompt_value=model_prompt_details["initial_prompt_value"],
final_prompt_value=model_prompt_details["final_prompt_value"],
messages=messages,
messages=prompt_messages,
)
elif text_completion_request: # handle `/completions` requests
ollama_prompt = get_str_from_messages(messages=messages)
ollama_prompt = get_str_from_messages(messages=prompt_messages)
else: # handle `/chat/completions` requests
modified_prompt: Final = ollama_pt(model=model, messages=messages)
modified_prompt: Final = ollama_pt(model=model, messages=prompt_messages)
if isinstance(modified_prompt, dict):
ollama_prompt, images = (
modified_prompt["prompt"],

12
litellm/main.py
View file

@ -180,7 +180,6 @@ from .litellm_core_utils.prompt_templates.common_utils import (
)
from .litellm_core_utils.prompt_templates.factory import (
custom_prompt,
function_call_prompt,
map_system_message_pt,
ollama_pt,
prompt_factory,
@ -221,6 +220,7 @@ from .llms.nvidia_riva.audio_transcription.transformation import (
NvidiaRivaAudioTranscriptionConfig,
)
from .llms.oci.chat.transformation import OCIChatConfig
from .llms.ollama.common_utils import resolve_ollama_tool_calling_provider
from .llms.ollama.completion import handler as ollama
from .llms.oobabooga.chat import oobabooga
from .llms.openai.completion.handler import OpenAITextCompletion
@ -5344,6 +5344,10 @@ def completion(
GenericLiteLLMParams(**_supplemental_provider_params) if _supplemental_provider_params else None
),
)
if tools or functions:
custom_llm_provider = resolve_ollama_tool_calling_provider( # rebind-ok: ollama tools use the chat adapter
custom_llm_provider, add_function_to_prompt=litellm.add_function_to_prompt
)
## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name
responses_api_model_info, model = responses_api_bridge_check(
@ -5501,12 +5505,6 @@ def completion(
provider_config=provider_config,
)
if litellm.add_function_to_prompt and optional_params.get(
"functions_unsupported_model", None
): # if user opts to add it to prompt, when API doesn't support function calling
functions_unsupported_model: Final = optional_params.pop("functions_unsupported_model")
messages = function_call_prompt(messages=messages, functions=functions_unsupported_model)
# For logging - save the values of the litellm-specific params passed in
litellm_params = get_litellm_params(
acompletion=acompletion,

48
litellm/utils.py
View file

@ -4154,54 +4154,6 @@ def pre_process_optional_params(passed_params: dict, non_default_params: dict, c
non_default_params=passed_params, optional_params=optional_params
)
## raise exception if function calling passed in for a provider that doesn't support it
if "functions" in non_default_params or "function_call" in non_default_params or "tools" in non_default_params:
if (
custom_llm_provider == "ollama"
and custom_llm_provider != "text-completion-openai"
and custom_llm_provider != "azure"
and custom_llm_provider != "vertex_ai"
and custom_llm_provider != "anyscale"
and custom_llm_provider != "together_ai"
and custom_llm_provider != "groq"
and custom_llm_provider != "nvidia_nim"
and custom_llm_provider != "cerebras"
and custom_llm_provider != "xai"
and custom_llm_provider != "ai21_chat"
and custom_llm_provider != "volcengine"
and custom_llm_provider != "deepseek"
and custom_llm_provider != "codestral"
and custom_llm_provider != "mistral"
and custom_llm_provider != "anthropic"
and custom_llm_provider != "cohere_chat"
and custom_llm_provider != "cohere"
and custom_llm_provider != "bedrock"
and custom_llm_provider != "ollama_chat"
and custom_llm_provider != "openrouter"
and custom_llm_provider != "vercel_ai_gateway"
and custom_llm_provider != "nebius"
and custom_llm_provider != "wandb"
and custom_llm_provider not in litellm.openai_compatible_providers
):
if custom_llm_provider == "ollama":
# ollama actually supports json output
optional_params["format"] = "json"
litellm.add_function_to_prompt = True # so that main.py adds the function call to the prompt
if "tools" in non_default_params:
optional_params["functions_unsupported_model"] = non_default_params.pop("tools")
non_default_params.pop("tool_choice", None) # causes ollama requests to hang
elif "functions" in non_default_params:
optional_params["functions_unsupported_model"] = non_default_params.pop("functions")
elif litellm.add_function_to_prompt: # if user opts to add it to prompt instead
optional_params["functions_unsupported_model"] = non_default_params.pop(
"tools", non_default_params.pop("functions", None)
)
else:
raise UnsupportedParamsError(
status_code=500,
message=f"Function calling is not supported by {custom_llm_provider}.",
)
return optional_params

35
tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
_convert_to_bedrock_tool_call_result,
anthropic_messages_pt,
convert_to_gemini_tool_call_result,
function_call_prompt,
make_valid_bedrock_tool_name,
ollama_pt,
sanitize_messages_for_tool_calling,
@ -3721,3 +3722,37 @@ def test_convert_to_anthropic_tool_invoke_keeps_paired_server_tool_use():
},
server_result,
]
FUNCTION_PROMPT_DESCRIPTIONS: Final = "\n{'name': 'graph_stats'}\n"
@pytest.mark.parametrize(
("messages", "expected_system_contents"),
[
([{"role": "user", "content": "hi"}], None),
([{"role": "system", "content": "Be brief."}, {"role": "user", "content": "hi"}], "Be brief. "),
(
[{"role": "system", "content": [{"type": "text", "text": "Be brief."}]}, {"role": "user", "content": "hi"}],
[{"type": "text", "text": "Be brief."}],
),
],
)
def test_function_call_prompt_returns_new_messages(messages, expected_system_contents):
original: Final = json.loads(json.dumps(messages))
result: Final = function_call_prompt(messages=messages, function_descriptions=FUNCTION_PROMPT_DESCRIPTIONS)
assert messages == original
system_messages: Final = [m for m in result if m["role"] == "system"]
assert len(system_messages) == 1
content: Final = system_messages[0]["content"]
prompt_text: Final = content if isinstance(content, str) else content[-1]["text"]
assert "Produce JSON OUTPUT ONLY" in prompt_text
assert "graph_stats" in prompt_text
if expected_system_contents is None:
assert result[:-1] == original
elif isinstance(expected_system_contents, str):
assert content.startswith(expected_system_contents)
else:
assert content[:-1] == expected_system_contents

20
tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py
View file

@ -944,3 +944,23 @@ class TestOllamaToolCallTransformation:
assert tool_msg["content"] == "Sunny, 72°F"
assert "tool_call_id" in tool_msg, "tool_call_id must be forwarded to Ollama"
assert tool_msg["tool_call_id"] == "call_abc123"
@pytest.mark.parametrize(
("api_base", "expected_url"),
[
(None, "http://localhost:11434/api/chat"),
("http://ollama.example:11434", "http://ollama.example:11434/api/chat"),
("http://ollama.example:11434/", "http://ollama.example:11434/api/chat"),
("http://ollama.example:11434/api/chat", "http://ollama.example:11434/api/chat"),
("http://ollama.example:11434/api/chat/", "http://ollama.example:11434/api/chat"),
("http://ollama.example:11434/api/generate", "http://ollama.example:11434/api/chat"),
("http://ollama.example:11434/prefix/api/generate/", "http://ollama.example:11434/prefix/api/chat"),
],
)
def test_get_complete_url_points_at_chat_endpoint(api_base, expected_url):
url = OllamaChatConfig().get_complete_url(
api_base=api_base, api_key=None, model="qwen3.8:27b", optional_params={}, litellm_params={}
)
assert url == expected_url

223
tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py
View file

@ -1,12 +1,13 @@
import json
from litellm._uuid import uuid
from typing import Final
from unittest.mock import MagicMock, patch
import httpx
import pytest
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm._uuid import uuid
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.ollama.completion.transformation import (
OllamaConfig,
OllamaTextCompletionResponseIterator,
@ -476,7 +477,7 @@ class TestOllamaTextCompletionResponseIterator:
# Updated to handle ModelResponseStream return type
assert isinstance(result, ModelResponseStream)
assert result.choices and result.choices[0].delta is not None
assert result.choices[0].delta.content == None
assert result.choices[0].delta.content is None
assert getattr(result.choices[0].delta, "reasoning_content", None) == ""
def test_chunk_parser_done_chunk(self):
@ -544,3 +545,219 @@ async def test_ollama_async_completion_inlines_remote_images_off_the_event_loop(
assert response.choices[0].message.content == "Green"
assert async_only_image_fetch.fetched == [image_url]
assert captured["body"]["images"] == [async_only_image_fetch.base64_png]
GRAPH_STATS_TOOLS = [
{
"type": "function",
"function": {
"name": "graph_stats",
"description": "Return node and edge counts of the code graph",
"parameters": {"type": "object", "properties": {}},
},
}
]
@pytest.mark.parametrize(
"api_base",
[
"http://ollama.example:11434",
"http://ollama.example:11434/",
"http://ollama.example:11434/api/generate",
"http://ollama.example:11434/api/generate/",
"http://ollama.example:11434/api/chat",
],
)
def test_ollama_tool_result_turn_is_sent_to_native_chat_api(api_base: str):
"""https://github.com/BerriAI/litellm/issues/40575"""
requests = []
def handle(request):
requests.append((request.url.path, json.loads(request.content)))
return httpx.Response(
200,
json={
"model": "qwen3.8:27b",
"message": {"role": "assistant", "content": "The graph has 190921 nodes."},
"done": True,
"done_reason": "stop",
"prompt_eval_count": 1,
"eval_count": 1,
},
)
response = litellm.completion(
model="ollama/qwen3.8:27b",
messages=[
{"role": "user", "content": "How many nodes does the graph have?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_1", "type": "function", "function": {"name": "graph_stats", "arguments": "{}"}}
],
},
{"role": "tool", "tool_call_id": "call_1", "name": "graph_stats", "content": '{"nodes": 190921}'},
],
tools=GRAPH_STATS_TOOLS,
api_base=api_base,
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))),
)
assert [path for path, _ in requests] == ["/api/chat"]
body = requests[0][1]
assert body["tools"] == GRAPH_STATS_TOOLS
assert "format" not in body
assert [m["role"] for m in body["messages"]] == ["user", "assistant", "tool"]
assert body["messages"][2]["content"] == '{"nodes": 190921}'
assert response.choices[0].message.content == "The graph has 190921 nodes."
assert response.choices[0].message.tool_calls is None
assert response.choices[0].finish_reason == "stop"
def test_ollama_streamed_tool_call_is_returned_as_tool_call():
"""https://github.com/BerriAI/litellm/issues/35711"""
chunks = [
{
"model": "qwen3.8:27b",
"message": {
"role": "assistant",
"content": "",
"tool_calls": [{"function": {"name": "graph_stats", "arguments": {}}}],
},
"done": False,
},
{
"model": "qwen3.8:27b",
"message": {"role": "assistant", "content": ""},
"done": True,
"done_reason": "stop",
"prompt_eval_count": 1,
"eval_count": 1,
},
]
def handle(request):
assert request.url.path == "/api/chat"
return httpx.Response(200, content="\n".join(json.dumps(chunk) for chunk in chunks).encode())
streamed = list(
litellm.completion(
model="ollama/qwen3.8:27b",
messages=[{"role": "user", "content": "How many nodes does the graph have?"}],
tools=GRAPH_STATS_TOOLS,
stream=True,
api_base="http://ollama.example:11434",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))),
)
)
tool_calls = [tool_call for chunk in streamed for tool_call in chunk.choices[0].delta.tool_calls or []]
assert [tool_call.function.name for tool_call in tool_calls] == ["graph_stats"]
assert "".join(chunk.choices[0].delta.content or "" for chunk in streamed) == ""
assert streamed[-1].choices[0].finish_reason == "tool_calls"
@pytest.mark.parametrize("empty_parameter", ["none", "tools", "functions"])
def test_ollama_empty_tools_preserve_generate_request(empty_parameter: str) -> None:
def handle(request: httpx.Request) -> httpx.Response:
body: Final = json.loads(request.content)
assert request.url.path == "/api/generate"
assert "format" not in body
assert "tools" not in body
return httpx.Response(200, json={"response": "Hello", "done": True})
response: Final = litellm.completion(
model="ollama/qwen3.8:27b",
messages=[{"role": "user", "content": "Hello"}],
tools=[] if empty_parameter == "tools" else None,
functions=[] if empty_parameter == "functions" else None,
api_base="http://ollama.example:11434/api/generate",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))),
)
assert response.choices[0].message.content == "Hello"
def test_ollama_native_tool_support_error_is_preserved() -> None:
def handle(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/api/chat"
return httpx.Response(400, json={"error": "model does not support tools"})
with pytest.raises(litellm.BadRequestError, match="does not support tools"):
litellm.completion(
model="ollama/qwen3.8:27b",
messages=[{"role": "user", "content": "Hello"}],
tools=GRAPH_STATS_TOOLS,
api_base="http://ollama.example:11434/api/generate",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))),
num_retries=0,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("legacy_functions", [False, True])
async def test_ollama_async_native_tools(legacy_functions: bool) -> None:
def handle(request: httpx.Request) -> httpx.Response:
body: Final = json.loads(request.content)
assert request.url.path == "/prefix/api/chat"
assert body["tools"] == GRAPH_STATS_TOOLS
return httpx.Response(
200,
json={
"model": "qwen3.8:27b",
"message": {"role": "assistant", "content": "Hello"},
"done": True,
"done_reason": "stop",
"prompt_eval_count": 1,
"eval_count": 1,
},
)
async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as client:
handler: Final = AsyncHTTPHandler()
await handler.client.aclose()
handler.client = client
response: Final = await litellm.acompletion(
model="ollama/qwen3.8:27b",
messages=[{"role": "user", "content": "Hello"}],
tools=None if legacy_functions else GRAPH_STATS_TOOLS,
functions=[GRAPH_STATS_TOOLS[0]["function"]] if legacy_functions else None,
api_base="http://ollama.example:11434/prefix/api/generate/",
client=handler,
)
assert response.choices[0].message.content == "Hello"
def test_ollama_add_function_to_prompt_keeps_legacy_json_emulation(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "add_function_to_prompt", True)
requests = []
def handle(request: httpx.Request) -> httpx.Response:
requests.append((request.url.path, json.loads(request.content)))
return httpx.Response(
200, json={"response": '{"name": "graph_stats", "arguments": {}}', "done": True, "prompt_eval_count": 1}
)
messages: Final = [
{"role": "system", "content": "You are a graph assistant."},
{"role": "user", "content": "How many nodes does the graph have?"},
]
response: Final = litellm.completion(
model="ollama/qwen3.8:27b",
messages=messages,
tools=GRAPH_STATS_TOOLS,
tool_choice="auto",
api_base="http://ollama.example:11434",
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handle))),
)
assert [path for path, _ in requests] == ["/api/generate"]
body: Final = requests[0][1]
assert body["format"] == "json"
assert "Produce JSON OUTPUT ONLY" in body["prompt"]
assert "graph_stats" in body["prompt"]
assert "prompted_functions" not in body["options"]
assert response.choices[0].message.tool_calls[0].function.name == "graph_stats"
assert response.choices[0].finish_reason == "tool_calls"

5
tests/test_litellm/test_system_message_format_bug.py
View file

@ -2,6 +2,7 @@
Test for GitHub issue #11267 - System message format issue with Ollama + tools
"""
import copy
from unittest.mock import patch
@ -49,6 +50,8 @@ def test_system_message_format_issue_reproduction():
}
]
original_messages = copy.deepcopy(messages)
response = completion(
model=model,
messages=messages,
@ -57,7 +60,7 @@ def test_system_message_format_issue_reproduction():
mock_response=True,
)
assert len(messages[1]["content"]) == 2
assert messages == original_messages
if __name__ == "__main__":