From 6a232c90e0865aa4c01f252986a4bfb1a4075795 Mon Sep 17 00:00:00 2001 From: Ted Swinyar <8061019+tedswinyar@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:13:40 -0800 Subject: [PATCH] fix(ollama_chat): give each streamed tool call its own index (#49183) * fix(ollama_chat): give each streamed tool call its own index Ollama streams parallel tool calls one per chunk with no top-level index, and the delta numbers the calls of each chunk from zero, so every call left the proxy as index 0 and OpenAI-style clients appended the arguments of each later call onto the first one. Number the calls with a counter that lives for the whole stream instead. Fixes #33678 * fix(ollama_chat): answer 400 when replayed tool call arguments are not JSON A chat history that already holds a merged tool call from an older stream used to fail with a 500 APIConnectionError. It now answers 400 naming the tool call. The streaming tests use Ollama's real tool call shape and drop a case the merge base already passed. * fix(ollama_chat): parse replayed tool call arguments with the shared repair parser Empty, null, and truncated arguments in a replayed assistant tool call now go through parse_tool_call_arguments, so they are forwarded as an object instead of answering 500. Arguments that still are not a JSON object answer 400 naming the call id. * fix(ollama_chat): send {} for null replayed arguments and keep argument text out of the repair warning Replayed tool call arguments of the JSON string "null" used to reach Ollama and return 200. The shared parser turns them into None, so they now go out as {} like empty or missing arguments instead of answering 400. The parser's repair warning logged up to 200 characters of the arguments. It now logs only the tool name, context, and length. * fix(ollama): keep replayed tool arguments out of a rejected replay's error chain * fix(ollama): keep the repair warning's argument preview unless message logging is off The shared repair warning lost its 200-character argument preview for every provider. It now logs the same preview as before by default, and logs redacted-by-litellm instead when turn_off_message_logging is on. * test(ollama): cover short and boundary-length repair warning previews --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../prompt_templates/tool_call_json_repair.py | 16 +- litellm/llms/ollama/chat/transformation.py | 34 ++++- ..._prompt_templates_tool_call_json_repair.py | 54 +++++++ .../ollama/test_ollama_chat_transformation.py | 137 +++++++++++++++++- 4 files changed, 231 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py b/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py index 317ffa7b3b1..ace6d25c9f6 100644 --- a/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py +++ b/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py @@ -3,10 +3,21 @@ from typing import Any, Final # noqa: TID251 # Existing annotations retained u from pydantic import TypeAdapter +import litellm from litellm import verbose_logger +from litellm.constants import REDACTED_BY_LITELLM from litellm.litellm_core_utils.core_helpers import as_str_mapping _DECODED_JSON: Final = TypeAdapter(object) +_ARGUMENT_PREVIEW_CHARS: Final = 200 + + +def _logged_arguments(arguments: str) -> str: + if litellm.turn_off_message_logging: + return REDACTED_BY_LITELLM + if len(arguments) <= _ARGUMENT_PREVIEW_CHARS: + return arguments + return f"{arguments[:_ARGUMENT_PREVIEW_CHARS]}..." def _attempt_json_repair(s: str) -> object | None: @@ -104,12 +115,11 @@ def parse_tool_call_arguments( repaired: Final = _attempt_json_repair(arguments) if repaired is not None: verbose_logger.warning( - "Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %.200s%s", + "Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %s", tool_name or "", context or "unknown context", len(arguments), - arguments, - "..." if len(arguments) > 200 else "", + _logged_arguments(arguments), ) return repaired diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index c6f3a7a8280..1e3e6186dd1 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -16,6 +16,9 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( extract_reasoning_content, ) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + parse_tool_call_arguments, +) from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.litellm_core_utils.url_utils import ensure_api_route @@ -44,6 +47,27 @@ else: LiteLLMLoggingObj = Any _REQUEST_PARAMS: Final = TypeAdapter(dict[str, object]) +_TOOL_CALL_ARGUMENTS: Final = TypeAdapter(dict[str, object]) + + +def _to_ollama_tool_call_function(model: str, tool_call: ChatCompletionAssistantToolCall) -> OllamaToolCallFunction: + function: Final = tool_call["function"] + name: Final = function.get("name") or "" + raw_arguments: Final[object] = function.get("arguments") + try: + parsed: Final[object] = ( + parse_tool_call_arguments(raw_arguments, tool_name=name, context="Ollama chat request") + if isinstance(raw_arguments, str) or raw_arguments is None + else raw_arguments + ) + arguments: Final = _TOOL_CALL_ARGUMENTS.validate_python({} if parsed is None else parsed) + except ValueError: + raise litellm.BadRequestError( + message=f"OllamaException - arguments of assistant tool call {tool_call.get('id')!r} are not a JSON object", + model=model, + llm_provider="ollama_chat", + ) from None + return OllamaToolCallFunction(name=name, arguments=arguments) class OllamaChatConfig(BaseConfig): @@ -247,12 +271,7 @@ class OllamaChatConfig(BaseConfig): tools: Final = optional_params.pop("tools", None) def transform_tool_call(typed_tool: ChatCompletionAssistantToolCall) -> OllamaToolCall: - function: Final = OllamaToolCallFunction( - name=typed_tool["function"].get("name") or "", - arguments=json.loads(typed_tool["function"]["arguments"]) - if "arguments" in typed_tool["function"] - else {}, - ) + function: Final = _to_ollama_tool_call_function(model, typed_tool) tool_id: Final = typed_tool.get("id") ollama_tool_call: Final = ( OllamaToolCall(function=function) if tool_id is None else OllamaToolCall(id=tool_id, function=function) @@ -442,6 +461,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): started_reasoning_content: bool = False finished_reasoning_content: bool = False seen_tool_calls: bool = False + _next_tool_call_index: int = 0 def _is_function_call_complete(self, function_args: str | dict) -> bool: if isinstance(function_args, dict): @@ -489,6 +509,8 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): if tool_calls is not None: self.seen_tool_calls = True for tool_call in tool_calls: + tool_call["index"] = self._next_tool_call_index + self._next_tool_call_index += 1 function_args = tool_call.get("function").get("arguments") if function_args is not None and len(function_args) > 0: is_function_call_complete = self._is_function_call_complete(function_args) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_tool_call_json_repair.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_tool_call_json_repair.py index 75f4e686137..4bd86d80c36 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_tool_call_json_repair.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_tool_call_json_repair.py @@ -1,7 +1,11 @@ import json +import logging +from typing import Final import pytest +import litellm +from litellm.constants import REDACTED_BY_LITELLM from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, salvage_concatenated_tool_arguments, @@ -115,3 +119,53 @@ def _distinct_json_objects(count: int) -> str: ) def test_salvage_concatenated_tool_arguments(raw: str, expected: tuple[dict[str, object], ...]) -> None: assert salvage_concatenated_tool_arguments(raw) == expected + + +def test_parse_tool_call_arguments_repair_warning_previews_the_first_200_characters( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + arguments: Final = '{"patient": "Jane Doe", "note": "' + "x" * 300 + '"' + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test") + + assert result == {"patient": "Jane Doe", "note": "x" * 300} + assert ( + f"Repaired truncated tool call arguments for tool 'lookup' (test). Original ({len(arguments)} chars): " + f"{arguments[:200]}..." in caplog.text + ) + assert arguments not in caplog.text + + +@pytest.mark.parametrize( + ("arguments", "expected"), + ( + ('{"patient": "Jane Doe"', {"patient": "Jane Doe"}), + ('{"note": "' + "x" * 189 + '"', {"note": "x" * 189}), + ), +) +def test_parse_tool_call_arguments_repair_warning_logs_arguments_up_to_200_characters_whole( + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + arguments: str, + expected: dict[str, str], +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test") + + assert result == expected + assert f"Original ({len(arguments)} chars): {arguments}\n" in caplog.text + + +def test_parse_tool_call_arguments_repair_warning_redacts_arguments_when_message_logging_is_off( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + arguments: Final = '{"patient": "Jane Doe"' + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test") + + assert result == {"patient": "Jane Doe"} + assert f"Original ({len(arguments)} chars): {REDACTED_BY_LITELLM}" in caplog.text + assert "Jane Doe" not in caplog.text diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index 98ca5aee407..36df47deaab 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -1,8 +1,10 @@ import asyncio import inspect +import itertools import logging import os import sys +import traceback from typing import Final, cast import pytest @@ -26,7 +28,14 @@ from unittest.mock import ANY, MagicMock, patch import litellm from litellm._logging import verbose_logger -from litellm.types.utils import Choices, EmbeddingResponse, Message, ModelResponse, ModelResponseStream +from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Choices, + EmbeddingResponse, + Message, + ModelResponse, + ModelResponseStream, +) from unittest import mock @@ -1043,6 +1052,132 @@ class TestOllamaToolCallTransformation: assert "tool_call_id" in tool_msg, "tool_call_id must be forwarded to Ollama" assert tool_msg["tool_call_id"] == "call_abc123" + @staticmethod + def _replayed_history(arguments: object) -> list[AllMessageValues]: + return cast( + list[AllMessageValues], + [ + {"role": "user", "content": "Read a.rs and b.rs"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": {"name": "read_file", "arguments": arguments}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_abc123", "content": "fn main() {}"}, + ], + ) + + @pytest.mark.parametrize( + ("arguments", "expected"), + [ + ("", {}), + (None, {}), + ("null", {}), + ('{"path": "a.rs"', {"path": "a.rs"}), + ('{"path": "a.rs", "limit": 2}', {"path": "a.rs", "limit": 2}), + ({"path": "a.rs"}, {"path": "a.rs"}), + ], + ids=["empty", "null", "null_string", "truncated", "non_string_value", "object"], + ) + def test_replayed_tool_call_arguments_are_forwarded_as_an_object(self, arguments: object, expected: object): + result: Final = OllamaChatConfig().transform_request( + model="qwen3:8b", + messages=self._replayed_history(arguments), + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["messages"][1]["tool_calls"] == [ + {"id": "call_abc123", "function": {"name": "read_file", "arguments": expected}} + ] + + @pytest.mark.parametrize( + "arguments", + ['{"path": "a.rs"}{"path": "b.rs"}', '["a.rs", "b.rs"]', '[["path", "a.rs"]]', "[]"], + ids=["glued", "array", "pairs", "empty_array"], + ) + def test_replayed_tool_call_arguments_that_are_not_an_object_answer_400_without_echoing_them( + self, arguments: str + ): + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.completion( + model="ollama_chat/qwen3:8b", + messages=self._replayed_history(arguments), + api_base="http://127.0.0.1:9", + ) + + assert exc_info.value.status_code == 400 + assert "call_abc123" in str(exc_info.value) + assert "a.rs" not in "".join(traceback.format_exception(exc_info.value)) + + +class TestOllamaStreamingParallelToolCallIndex: + """Regression for https://github.com/BerriAI/litellm/issues/33678. + + Ollama streams each parallel tool call in its own chunk without an OpenAI-style top-level + index. Every delta used to carry index 0, so index-accumulating clients appended the + arguments of every later call onto the first one. + """ + + @staticmethod + def _tool_call(index: int, command: str) -> dict[str, object]: + return { + "id": f"call_{index}", + "function": {"index": index, "name": "bash", "arguments": {"command": command}}, + } + + @staticmethod + def _chunk(*tool_calls: dict[str, object]) -> dict[str, object]: + return { + "model": "qwen3:8b", + "message": {"role": "assistant", "content": "", "tool_calls": list(tool_calls)}, + "done": False, + } + + @staticmethod + def _parsed_tool_calls( + iterator: OllamaChatCompletionResponseIterator, *chunks: dict[str, object] + ) -> tuple[ChatCompletionDeltaToolCall, ...]: + return tuple( + itertools.chain.from_iterable( + iterator.chunk_parser(chunk).choices[0].delta.tool_calls or () for chunk in chunks + ) + ) + + def test_tool_calls_in_separate_chunks_get_distinct_sequential_indices(self): + iterator: Final = OllamaChatCompletionResponseIterator(streaming_response=iter([]), sync_stream=True) + chunks: Final = [ + self._chunk(self._tool_call(index, command)) + for index, command in enumerate(("date", "uname -a", "whoami")) + ] + + tool_calls: Final = self._parsed_tool_calls(iterator, *chunks) + + assert [tool_call.index for tool_call in tool_calls] == [0, 1, 2] + assert [json.loads(tool_call.function.arguments) for tool_call in tool_calls] == [ + {"command": "date"}, + {"command": "uname -a"}, + {"command": "whoami"}, + ] + + def test_index_counter_continues_across_chunks(self): + iterator: Final = OllamaChatCompletionResponseIterator(streaming_response=iter([]), sync_stream=True) + chunks: Final = [ + self._chunk(self._tool_call(0, "date"), self._tool_call(1, "uname -a")), + self._chunk(self._tool_call(2, "whoami")), + ] + + tool_calls: Final = self._parsed_tool_calls(iterator, *chunks) + + assert [tool_call.index for tool_call in tool_calls] == [0, 1, 2] + class TestOllamaStreamingUsage: @staticmethod