scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 280

fix(ollama_chat): give each streamed tool call its own index (#49183)

Browse source 
* fix(ollama_chat): give each streamed tool call its own index

Ollama streams parallel tool calls one per chunk with no top-level index, and the delta
numbers the calls of each chunk from zero, so every call left the proxy as index 0 and
OpenAI-style clients appended the arguments of each later call onto the first one. Number
the calls with a counter that lives for the whole stream instead.

Fixes #33678

* fix(ollama_chat): answer 400 when replayed tool call arguments are not JSON

A chat history that already holds a merged tool call from an older stream
used to fail with a 500 APIConnectionError. It now answers 400 naming the
tool call. The streaming tests use Ollama's real tool call shape and drop
a case the merge base already passed.

* fix(ollama_chat): parse replayed tool call arguments with the shared repair parser

Empty, null, and truncated arguments in a replayed assistant tool call now go through parse_tool_call_arguments, so they are forwarded as an object instead of answering 500. Arguments that still are not a JSON object answer 400 naming the call id.

* fix(ollama_chat): send {} for null replayed arguments and keep argument text out of the repair warning

Replayed tool call arguments of the JSON string "null" used to reach Ollama and return 200. The shared parser turns them into None, so they now go out as {} like empty or missing arguments instead of answering 400.

The parser's repair warning logged up to 200 characters of the arguments. It now logs only the tool name, context, and length.

* fix(ollama): keep replayed tool arguments out of a rejected replay's error chain

* fix(ollama): keep the repair warning's argument preview unless message logging is off

The shared repair warning lost its 200-character argument preview for every provider. It now logs the same preview as before by default, and logs redacted-by-litellm instead when turn_off_message_logging is on.

* test(ollama): cover short and boundary-length repair warning previews

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Ted Swinyar 2026-11-09 14:13:40 -08:00 • committed by GitHub
parent 9de20b6cf8
commit 6a232c90e0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 231 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py
View file

@ -3,10 +3,21 @@ from typing import Any, Final # noqa: TID251 # Existing annotations retained u
from pydantic import TypeAdapter
import litellm
from litellm import verbose_logger
from litellm.constants import REDACTED_BY_LITELLM
from litellm.litellm_core_utils.core_helpers import as_str_mapping
_DECODED_JSON: Final = TypeAdapter(object)
_ARGUMENT_PREVIEW_CHARS: Final = 200
def _logged_arguments(arguments: str) -> str:
if litellm.turn_off_message_logging:
return REDACTED_BY_LITELLM
if len(arguments) <= _ARGUMENT_PREVIEW_CHARS:
return arguments
return f"{arguments[:_ARGUMENT_PREVIEW_CHARS]}..."
def _attempt_json_repair(s: str) -> object | None:
@ -104,12 +115,11 @@ def parse_tool_call_arguments(
repaired: Final = _attempt_json_repair(arguments)
if repaired is not None:
verbose_logger.warning(
"Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %.200s%s",
"Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %s",
tool_name or "<unknown>",
context or "unknown context",
len(arguments),
arguments,
"..." if len(arguments) > 200 else "",
_logged_arguments(arguments),
)
return repaired

34
litellm/llms/ollama/chat/transformation.py
View file

@ -16,6 +16,9 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
extract_reasoning_content,
)
from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
parse_tool_call_arguments,
)
from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE
from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
from litellm.litellm_core_utils.url_utils import ensure_api_route
@ -44,6 +47,27 @@ else:
LiteLLMLoggingObj = Any
_REQUEST_PARAMS: Final = TypeAdapter(dict[str, object])
_TOOL_CALL_ARGUMENTS: Final = TypeAdapter(dict[str, object])
def _to_ollama_tool_call_function(model: str, tool_call: ChatCompletionAssistantToolCall) -> OllamaToolCallFunction:
function: Final = tool_call["function"]
name: Final = function.get("name") or ""
raw_arguments: Final[object] = function.get("arguments")
try:
parsed: Final[object] = (
parse_tool_call_arguments(raw_arguments, tool_name=name, context="Ollama chat request")
if isinstance(raw_arguments, str) or raw_arguments is None
else raw_arguments
)
arguments: Final = _TOOL_CALL_ARGUMENTS.validate_python({} if parsed is None else parsed)
except ValueError:
raise litellm.BadRequestError(
message=f"OllamaException - arguments of assistant tool call {tool_call.get('id')!r} are not a JSON object",
model=model,
llm_provider="ollama_chat",
) from None
return OllamaToolCallFunction(name=name, arguments=arguments)
class OllamaChatConfig(BaseConfig):
@ -247,12 +271,7 @@ class OllamaChatConfig(BaseConfig):
tools: Final = optional_params.pop("tools", None)
def transform_tool_call(typed_tool: ChatCompletionAssistantToolCall) -> OllamaToolCall:
function: Final = OllamaToolCallFunction(
name=typed_tool["function"].get("name") or "",
arguments=json.loads(typed_tool["function"]["arguments"])
if "arguments" in typed_tool["function"]
else {},
)
function: Final = _to_ollama_tool_call_function(model, typed_tool)
tool_id: Final = typed_tool.get("id")
ollama_tool_call: Final = (
OllamaToolCall(function=function) if tool_id is None else OllamaToolCall(id=tool_id, function=function)
@ -442,6 +461,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
started_reasoning_content: bool = False
finished_reasoning_content: bool = False
seen_tool_calls: bool = False
_next_tool_call_index: int = 0
def _is_function_call_complete(self, function_args: str | dict) -> bool:
if isinstance(function_args, dict):
@ -489,6 +509,8 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
if tool_calls is not None:
self.seen_tool_calls = True
for tool_call in tool_calls:
tool_call["index"] = self._next_tool_call_index
self._next_tool_call_index += 1
function_args = tool_call.get("function").get("arguments")
if function_args is not None and len(function_args) > 0:
is_function_call_complete = self._is_function_call_complete(function_args)

54
tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_tool_call_json_repair.py
View file

@ -1,7 +1,11 @@
import json
import logging
from typing import Final
import pytest
import litellm
from litellm.constants import REDACTED_BY_LITELLM
from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
parse_tool_call_arguments,
salvage_concatenated_tool_arguments,
@ -115,3 +119,53 @@ def _distinct_json_objects(count: int) -> str:
)
def test_salvage_concatenated_tool_arguments(raw: str, expected: tuple[dict[str, object], ...]) -> None:
assert salvage_concatenated_tool_arguments(raw) == expected
def test_parse_tool_call_arguments_repair_warning_previews_the_first_200_characters(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
arguments: Final = '{"patient": "Jane Doe", "note": "' + "x" * 300 + '"'
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test")
assert result == {"patient": "Jane Doe", "note": "x" * 300}
assert (
f"Repaired truncated tool call arguments for tool 'lookup' (test). Original ({len(arguments)} chars): "
f"{arguments[:200]}..." in caplog.text
)
assert arguments not in caplog.text
@pytest.mark.parametrize(
("arguments", "expected"),
(
('{"patient": "Jane Doe"', {"patient": "Jane Doe"}),
('{"note": "' + "x" * 189 + '"', {"note": "x" * 189}),
),
)
def test_parse_tool_call_arguments_repair_warning_logs_arguments_up_to_200_characters_whole(
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
arguments: str,
expected: dict[str, str],
) -> None:
monkeypatch.setattr(litellm, "turn_off_message_logging", False)
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test")
assert result == expected
assert f"Original ({len(arguments)} chars): {arguments}\n" in caplog.text
def test_parse_tool_call_arguments_repair_warning_redacts_arguments_when_message_logging_is_off(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "turn_off_message_logging", True)
arguments: Final = '{"patient": "Jane Doe"'
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test")
assert result == {"patient": "Jane Doe"}
assert f"Original ({len(arguments)} chars): {REDACTED_BY_LITELLM}" in caplog.text
assert "Jane Doe" not in caplog.text

137
tests/unit/llms/ollama/test_ollama_chat_transformation.py
View file

@ -1,8 +1,10 @@
import asyncio
import inspect
import itertools
import logging
import os
import sys
import traceback
from typing import Final, cast
import pytest
@ -26,7 +28,14 @@ from unittest.mock import ANY, MagicMock, patch
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import Choices, EmbeddingResponse, Message, ModelResponse, ModelResponseStream
from litellm.types.utils import (
ChatCompletionDeltaToolCall,
Choices,
EmbeddingResponse,
Message,
ModelResponse,
ModelResponseStream,
)
from unittest import mock
@ -1043,6 +1052,132 @@ class TestOllamaToolCallTransformation:
assert "tool_call_id" in tool_msg, "tool_call_id must be forwarded to Ollama"
assert tool_msg["tool_call_id"] == "call_abc123"
@staticmethod
def _replayed_history(arguments: object) -> list[AllMessageValues]:
return cast(
list[AllMessageValues],
[
{"role": "user", "content": "Read a.rs and b.rs"},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_abc123",
"type": "function",
"function": {"name": "read_file", "arguments": arguments},
}
],
},
{"role": "tool", "tool_call_id": "call_abc123", "content": "fn main() {}"},
],
)
@pytest.mark.parametrize(
("arguments", "expected"),
[
("", {}),
(None, {}),
("null", {}),
('{"path": "a.rs"', {"path": "a.rs"}),
('{"path": "a.rs", "limit": 2}', {"path": "a.rs", "limit": 2}),
({"path": "a.rs"}, {"path": "a.rs"}),
],
ids=["empty", "null", "null_string", "truncated", "non_string_value", "object"],
)
def test_replayed_tool_call_arguments_are_forwarded_as_an_object(self, arguments: object, expected: object):
result: Final = OllamaChatConfig().transform_request(
model="qwen3:8b",
messages=self._replayed_history(arguments),
optional_params={},
litellm_params={},
headers={},
)
assert result["messages"][1]["tool_calls"] == [
{"id": "call_abc123", "function": {"name": "read_file", "arguments": expected}}
]
@pytest.mark.parametrize(
"arguments",
['{"path": "a.rs"}{"path": "b.rs"}', '["a.rs", "b.rs"]', '[["path", "a.rs"]]', "[]"],
ids=["glued", "array", "pairs", "empty_array"],
)
def test_replayed_tool_call_arguments_that_are_not_an_object_answer_400_without_echoing_them(
self, arguments: str
):
with pytest.raises(litellm.BadRequestError) as exc_info:
litellm.completion(
model="ollama_chat/qwen3:8b",
messages=self._replayed_history(arguments),
api_base="http://127.0.0.1:9",
)
assert exc_info.value.status_code == 400
assert "call_abc123" in str(exc_info.value)
assert "a.rs" not in "".join(traceback.format_exception(exc_info.value))
class TestOllamaStreamingParallelToolCallIndex:
"""Regression for https://github.com/BerriAI/litellm/issues/33678.
Ollama streams each parallel tool call in its own chunk without an OpenAI-style top-level
index. Every delta used to carry index 0, so index-accumulating clients appended the
arguments of every later call onto the first one.
"""
@staticmethod
def _tool_call(index: int, command: str) -> dict[str, object]:
return {
"id": f"call_{index}",
"function": {"index": index, "name": "bash", "arguments": {"command": command}},
}
@staticmethod
def _chunk(*tool_calls: dict[str, object]) -> dict[str, object]:
return {
"model": "qwen3:8b",
"message": {"role": "assistant", "content": "", "tool_calls": list(tool_calls)},
"done": False,
}
@staticmethod
def _parsed_tool_calls(
iterator: OllamaChatCompletionResponseIterator, *chunks: dict[str, object]
) -> tuple[ChatCompletionDeltaToolCall, ...]:
return tuple(
itertools.chain.from_iterable(
iterator.chunk_parser(chunk).choices[0].delta.tool_calls or () for chunk in chunks
)
)
def test_tool_calls_in_separate_chunks_get_distinct_sequential_indices(self):
iterator: Final = OllamaChatCompletionResponseIterator(streaming_response=iter([]), sync_stream=True)
chunks: Final = [
self._chunk(self._tool_call(index, command))
for index, command in enumerate(("date", "uname -a", "whoami"))
]
tool_calls: Final = self._parsed_tool_calls(iterator, *chunks)
assert [tool_call.index for tool_call in tool_calls] == [0, 1, 2]
assert [json.loads(tool_call.function.arguments) for tool_call in tool_calls] == [
{"command": "date"},
{"command": "uname -a"},
{"command": "whoami"},
]
def test_index_counter_continues_across_chunks(self):
iterator: Final = OllamaChatCompletionResponseIterator(streaming_response=iter([]), sync_stream=True)
chunks: Final = [
self._chunk(self._tool_call(0, "date"), self._tool_call(1, "uname -a")),
self._chunk(self._tool_call(2, "whoami")),
]
tool_calls: Final = self._parsed_tool_calls(iterator, *chunks)
assert [tool_call.index for tool_call in tool_calls] == [0, 1, 2]
class TestOllamaStreamingUsage:
@staticmethod