diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index f9c4ebca350..1de66ff8799 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -20,6 +20,7 @@ from pydantic import TypeAdapter import litellm from litellm import verbose_logger +from litellm.constants import REDACTED_BY_LITELLM from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.router_utils.batch_utils import InMemoryFile from litellm.types.llms.openai import ( @@ -2752,6 +2753,15 @@ def merge_consecutive_system_messages( _DECODED_JSON: Final = TypeAdapter(object) +_ARGUMENT_PREVIEW_CHARS: Final = 200 + + +def _logged_arguments(arguments: str) -> str: + if litellm.turn_off_message_logging: + return REDACTED_BY_LITELLM + if len(arguments) <= _ARGUMENT_PREVIEW_CHARS: + return arguments + return f"{arguments[:_ARGUMENT_PREVIEW_CHARS]}..." def _attempt_json_repair(s: str) -> object | None: @@ -2849,12 +2859,11 @@ def parse_tool_call_arguments( repaired: Final = _attempt_json_repair(arguments) if repaired is not None: verbose_logger.warning( - "Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %.200s%s", + "Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %s", tool_name or "", context or "unknown context", len(arguments), - arguments, - "..." if len(arguments) > 200 else "", + _logged_arguments(arguments), ) return repaired diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 46b4caad43f..f10f56ebd92 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -13,6 +13,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_images_from_message, extract_reasoning_content, + parse_tool_call_arguments, ) from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator @@ -40,6 +41,27 @@ else: LiteLLMLoggingObj = Any _REQUEST_PARAMS: Final = TypeAdapter(dict[str, object]) +_TOOL_CALL_ARGUMENTS: Final = TypeAdapter(dict[str, object]) + + +def _to_ollama_tool_call_function(model: str, tool_call: ChatCompletionAssistantToolCall) -> OllamaToolCallFunction: + function: Final = tool_call["function"] + name: Final = function.get("name") or "" + raw_arguments: Final[object] = function.get("arguments") + try: + parsed: Final[object] = ( + parse_tool_call_arguments(raw_arguments, tool_name=name, context="Ollama chat request") + if isinstance(raw_arguments, str) or raw_arguments is None + else raw_arguments + ) + arguments: Final = _TOOL_CALL_ARGUMENTS.validate_python({} if parsed is None else parsed) + except ValueError: + raise litellm.BadRequestError( + message=f"OllamaException - arguments of assistant tool call {tool_call.get('id')!r} are not a JSON object", + model=model, + llm_provider="ollama_chat", + ) from None + return OllamaToolCallFunction(name=name, arguments=arguments) class OllamaChatConfig(BaseConfig): @@ -261,16 +283,7 @@ class OllamaChatConfig(BaseConfig): for tool in tool_calls: typed_tool = ChatCompletionAssistantToolCall(**tool) if typed_tool["type"] == "function": - arguments = {} - if "arguments" in typed_tool["function"]: - arguments = json.loads(typed_tool["function"]["arguments"]) - ollama_tool_call = OllamaToolCall( - function=OllamaToolCallFunction( - name=typed_tool["function"].get("name") or "", - arguments=arguments, - ) - ) - new_tools.append(ollama_tool_call) + new_tools.append(OllamaToolCall(function=_to_ollama_tool_call_function(model, typed_tool))) reasoning_content, parsed_content = extract_reasoning_content(cast(dict, m)) content_str = convert_content_list_to_str(cast(AllMessageValues, m)) images = extract_images_from_message(cast(AllMessageValues, m)) @@ -440,6 +453,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): started_reasoning_content: bool = False finished_reasoning_content: bool = False seen_tool_calls: bool = False + _next_tool_call_index: int = 0 def _is_function_call_complete(self, function_args: str | dict) -> bool: if isinstance(function_args, dict): @@ -487,6 +501,8 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): if tool_calls is not None: self.seen_tool_calls = True for tool_call in tool_calls: + tool_call["index"] = self._next_tool_call_index + self._next_tool_call_index += 1 function_args = tool_call.get("function").get("arguments") if function_args is not None and len(function_args) > 0: is_function_call_complete = self._is_function_call_complete(function_args) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index c08eeea4bb9..b8c9888e8c8 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -2,6 +2,7 @@ import base64 import copy import functools import json +import logging import os import sys from collections.abc import Mapping @@ -9,6 +10,8 @@ from typing import Final, cast import pytest +import litellm +from litellm.constants import REDACTED_BY_LITELLM from litellm.litellm_core_utils.prompt_templates.common_utils import ( ENCRYPTED_REASONING_SIGNATURE_PREFIX, TOOL_RESULT_IMAGE_BOUNDARY, @@ -2439,3 +2442,53 @@ def test_schema_has_unresolvable_ref_flags_refs_that_cannot_be_inlined(schema, u def test_schema_size_counts_every_written_object_once(schema, properties, branches): assert schema_property_count(schema) == properties assert schema_grammar_branch_count(schema) == branches + + +def test_parse_tool_call_arguments_repair_warning_previews_the_first_200_characters( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + arguments: Final = '{"patient": "Jane Doe", "note": "' + "x" * 300 + '"' + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test") + + assert result == {"patient": "Jane Doe", "note": "x" * 300} + assert ( + f"Repaired truncated tool call arguments for tool 'lookup' (test). Original ({len(arguments)} chars): " + f"{arguments[:200]}..." in caplog.text + ) + assert arguments not in caplog.text + + +@pytest.mark.parametrize( + ("arguments", "expected"), + ( + ('{"patient": "Jane Doe"', {"patient": "Jane Doe"}), + ('{"note": "' + "x" * 189 + '"', {"note": "x" * 189}), + ), +) +def test_parse_tool_call_arguments_repair_warning_logs_arguments_up_to_200_characters_whole( + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + arguments: str, + expected: dict[str, str], +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test") + + assert result == expected + assert f"Original ({len(arguments)} chars): {arguments}\n" in caplog.text + + +def test_parse_tool_call_arguments_repair_warning_redacts_arguments_when_message_logging_is_off( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + arguments: Final = '{"patient": "Jane Doe"' + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test") + + assert result == {"patient": "Jane Doe"} + assert f"Original ({len(arguments)} chars): {REDACTED_BY_LITELLM}" in caplog.text + assert "Jane Doe" not in caplog.text diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index ae75c40d32a..552ae778409 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -1,7 +1,9 @@ import asyncio import inspect +import itertools import os import sys +import traceback from typing import Final, cast import pytest @@ -24,7 +26,14 @@ import json from unittest.mock import ANY, MagicMock, patch import litellm -from litellm.types.utils import Choices, EmbeddingResponse, Message, ModelResponse, ModelResponseStream +from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Choices, + EmbeddingResponse, + Message, + ModelResponse, + ModelResponseStream, +) from unittest import mock @@ -948,6 +957,130 @@ class TestOllamaToolCallTransformation: assert "tool_call_id" in tool_msg, "tool_call_id must be forwarded to Ollama" assert tool_msg["tool_call_id"] == "call_abc123" + @staticmethod + def _replayed_history(arguments: object) -> list[AllMessageValues]: + return cast( + list[AllMessageValues], + [ + {"role": "user", "content": "Read a.rs and b.rs"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": {"name": "read_file", "arguments": arguments}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_abc123", "content": "fn main() {}"}, + ], + ) + + @pytest.mark.parametrize( + ("arguments", "expected"), + [ + ("", {}), + (None, {}), + ("null", {}), + ('{"path": "a.rs"', {"path": "a.rs"}), + ('{"path": "a.rs", "limit": 2}', {"path": "a.rs", "limit": 2}), + ({"path": "a.rs"}, {"path": "a.rs"}), + ], + ids=["empty", "null", "null_string", "truncated", "non_string_value", "object"], + ) + def test_replayed_tool_call_arguments_are_forwarded_as_an_object(self, arguments: object, expected: object): + result: Final = OllamaChatConfig().transform_request( + model="qwen3:8b", + messages=self._replayed_history(arguments), + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["messages"][1]["tool_calls"] == [{"function": {"name": "read_file", "arguments": expected}}] + + @pytest.mark.parametrize( + "arguments", + ['{"path": "a.rs"}{"path": "b.rs"}', '["a.rs", "b.rs"]', '[["path", "a.rs"]]', "[]"], + ids=["glued", "array", "pairs", "empty_array"], + ) + def test_replayed_tool_call_arguments_that_are_not_an_object_answer_400_without_echoing_them( + self, arguments: str + ): + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.completion( + model="ollama_chat/qwen3:8b", + messages=self._replayed_history(arguments), + api_base="http://127.0.0.1:9", + ) + + assert exc_info.value.status_code == 400 + assert "call_abc123" in str(exc_info.value) + assert "a.rs" not in "".join(traceback.format_exception(exc_info.value)) + + +class TestOllamaStreamingParallelToolCallIndex: + """Regression for https://github.com/BerriAI/litellm/issues/33678. + + Ollama streams each parallel tool call in its own chunk without an OpenAI-style top-level + index. Every delta used to carry index 0, so index-accumulating clients appended the + arguments of every later call onto the first one. + """ + + @staticmethod + def _tool_call(index: int, command: str) -> dict[str, object]: + return { + "id": f"call_{index}", + "function": {"index": index, "name": "bash", "arguments": {"command": command}}, + } + + @staticmethod + def _chunk(*tool_calls: dict[str, object]) -> dict[str, object]: + return { + "model": "qwen3:8b", + "message": {"role": "assistant", "content": "", "tool_calls": list(tool_calls)}, + "done": False, + } + + @staticmethod + def _parsed_tool_calls( + iterator: OllamaChatCompletionResponseIterator, *chunks: dict[str, object] + ) -> tuple[ChatCompletionDeltaToolCall, ...]: + return tuple( + itertools.chain.from_iterable( + iterator.chunk_parser(chunk).choices[0].delta.tool_calls or () for chunk in chunks + ) + ) + + def test_tool_calls_in_separate_chunks_get_distinct_sequential_indices(self): + iterator: Final = OllamaChatCompletionResponseIterator(streaming_response=iter([]), sync_stream=True) + chunks: Final = [ + self._chunk(self._tool_call(index, command)) + for index, command in enumerate(("date", "uname -a", "whoami")) + ] + + tool_calls: Final = self._parsed_tool_calls(iterator, *chunks) + + assert [tool_call.index for tool_call in tool_calls] == [0, 1, 2] + assert [json.loads(tool_call.function.arguments) for tool_call in tool_calls] == [ + {"command": "date"}, + {"command": "uname -a"}, + {"command": "whoami"}, + ] + + def test_index_counter_continues_across_chunks(self): + iterator: Final = OllamaChatCompletionResponseIterator(streaming_response=iter([]), sync_stream=True) + chunks: Final = [ + self._chunk(self._tool_call(0, "date"), self._tool_call(1, "uname -a")), + self._chunk(self._tool_call(2, "whoami")), + ] + + tool_calls: Final = self._parsed_tool_calls(iterator, *chunks) + + assert [tool_call.index for tool_call in tool_calls] == [0, 1, 2] + class TestOllamaStreamingUsage: @staticmethod