diff --git a/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py b/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py index 317ffa7b3b1..ace6d25c9f6 100644 --- a/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py +++ b/litellm/litellm_core_utils/prompt_templates/tool_call_json_repair.py @@ -3,10 +3,21 @@ from typing import Any, Final # noqa: TID251 # Existing annotations retained u from pydantic import TypeAdapter +import litellm from litellm import verbose_logger +from litellm.constants import REDACTED_BY_LITELLM from litellm.litellm_core_utils.core_helpers import as_str_mapping _DECODED_JSON: Final = TypeAdapter(object) +_ARGUMENT_PREVIEW_CHARS: Final = 200 + + +def _logged_arguments(arguments: str) -> str: + if litellm.turn_off_message_logging: + return REDACTED_BY_LITELLM + if len(arguments) <= _ARGUMENT_PREVIEW_CHARS: + return arguments + return f"{arguments[:_ARGUMENT_PREVIEW_CHARS]}..." def _attempt_json_repair(s: str) -> object | None: @@ -104,12 +115,11 @@ def parse_tool_call_arguments( repaired: Final = _attempt_json_repair(arguments) if repaired is not None: verbose_logger.warning( - "Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %.200s%s", + "Repaired truncated tool call arguments for tool '%s' (%s). Original (%d chars): %s", tool_name or "", context or "unknown context", len(arguments), - arguments, - "..." if len(arguments) > 200 else "", + _logged_arguments(arguments), ) return repaired diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index c6f3a7a8280..1e3e6186dd1 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -16,6 +16,9 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( extract_reasoning_content, ) +from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( + parse_tool_call_arguments, +) from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp from litellm.litellm_core_utils.url_utils import ensure_api_route @@ -44,6 +47,27 @@ else: LiteLLMLoggingObj = Any _REQUEST_PARAMS: Final = TypeAdapter(dict[str, object]) +_TOOL_CALL_ARGUMENTS: Final = TypeAdapter(dict[str, object]) + + +def _to_ollama_tool_call_function(model: str, tool_call: ChatCompletionAssistantToolCall) -> OllamaToolCallFunction: + function: Final = tool_call["function"] + name: Final = function.get("name") or "" + raw_arguments: Final[object] = function.get("arguments") + try: + parsed: Final[object] = ( + parse_tool_call_arguments(raw_arguments, tool_name=name, context="Ollama chat request") + if isinstance(raw_arguments, str) or raw_arguments is None + else raw_arguments + ) + arguments: Final = _TOOL_CALL_ARGUMENTS.validate_python({} if parsed is None else parsed) + except ValueError: + raise litellm.BadRequestError( + message=f"OllamaException - arguments of assistant tool call {tool_call.get('id')!r} are not a JSON object", + model=model, + llm_provider="ollama_chat", + ) from None + return OllamaToolCallFunction(name=name, arguments=arguments) class OllamaChatConfig(BaseConfig): @@ -247,12 +271,7 @@ class OllamaChatConfig(BaseConfig): tools: Final = optional_params.pop("tools", None) def transform_tool_call(typed_tool: ChatCompletionAssistantToolCall) -> OllamaToolCall: - function: Final = OllamaToolCallFunction( - name=typed_tool["function"].get("name") or "", - arguments=json.loads(typed_tool["function"]["arguments"]) - if "arguments" in typed_tool["function"] - else {}, - ) + function: Final = _to_ollama_tool_call_function(model, typed_tool) tool_id: Final = typed_tool.get("id") ollama_tool_call: Final = ( OllamaToolCall(function=function) if tool_id is None else OllamaToolCall(id=tool_id, function=function) @@ -442,6 +461,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): started_reasoning_content: bool = False finished_reasoning_content: bool = False seen_tool_calls: bool = False + _next_tool_call_index: int = 0 def _is_function_call_complete(self, function_args: str | dict) -> bool: if isinstance(function_args, dict): @@ -489,6 +509,8 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): if tool_calls is not None: self.seen_tool_calls = True for tool_call in tool_calls: + tool_call["index"] = self._next_tool_call_index + self._next_tool_call_index += 1 function_args = tool_call.get("function").get("arguments") if function_args is not None and len(function_args) > 0: is_function_call_complete = self._is_function_call_complete(function_args) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_tool_call_json_repair.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_tool_call_json_repair.py index 75f4e686137..4bd86d80c36 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_tool_call_json_repair.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_tool_call_json_repair.py @@ -1,7 +1,11 @@ import json +import logging +from typing import Final import pytest +import litellm +from litellm.constants import REDACTED_BY_LITELLM from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, salvage_concatenated_tool_arguments, @@ -115,3 +119,53 @@ def _distinct_json_objects(count: int) -> str: ) def test_salvage_concatenated_tool_arguments(raw: str, expected: tuple[dict[str, object], ...]) -> None: assert salvage_concatenated_tool_arguments(raw) == expected + + +def test_parse_tool_call_arguments_repair_warning_previews_the_first_200_characters( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + arguments: Final = '{"patient": "Jane Doe", "note": "' + "x" * 300 + '"' + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test") + + assert result == {"patient": "Jane Doe", "note": "x" * 300} + assert ( + f"Repaired truncated tool call arguments for tool 'lookup' (test). Original ({len(arguments)} chars): " + f"{arguments[:200]}..." in caplog.text + ) + assert arguments not in caplog.text + + +@pytest.mark.parametrize( + ("arguments", "expected"), + ( + ('{"patient": "Jane Doe"', {"patient": "Jane Doe"}), + ('{"note": "' + "x" * 189 + '"', {"note": "x" * 189}), + ), +) +def test_parse_tool_call_arguments_repair_warning_logs_arguments_up_to_200_characters_whole( + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + arguments: str, + expected: dict[str, str], +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test") + + assert result == expected + assert f"Original ({len(arguments)} chars): {arguments}\n" in caplog.text + + +def test_parse_tool_call_arguments_repair_warning_redacts_arguments_when_message_logging_is_off( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + arguments: Final = '{"patient": "Jane Doe"' + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = parse_tool_call_arguments(arguments, tool_name="lookup", context="test") + + assert result == {"patient": "Jane Doe"} + assert f"Original ({len(arguments)} chars): {REDACTED_BY_LITELLM}" in caplog.text + assert "Jane Doe" not in caplog.text diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index 98ca5aee407..36df47deaab 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -1,8 +1,10 @@ import asyncio import inspect +import itertools import logging import os import sys +import traceback from typing import Final, cast import pytest @@ -26,7 +28,14 @@ from unittest.mock import ANY, MagicMock, patch import litellm from litellm._logging import verbose_logger -from litellm.types.utils import Choices, EmbeddingResponse, Message, ModelResponse, ModelResponseStream +from litellm.types.utils import ( + ChatCompletionDeltaToolCall, + Choices, + EmbeddingResponse, + Message, + ModelResponse, + ModelResponseStream, +) from unittest import mock @@ -1043,6 +1052,132 @@ class TestOllamaToolCallTransformation: assert "tool_call_id" in tool_msg, "tool_call_id must be forwarded to Ollama" assert tool_msg["tool_call_id"] == "call_abc123" + @staticmethod + def _replayed_history(arguments: object) -> list[AllMessageValues]: + return cast( + list[AllMessageValues], + [ + {"role": "user", "content": "Read a.rs and b.rs"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": {"name": "read_file", "arguments": arguments}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_abc123", "content": "fn main() {}"}, + ], + ) + + @pytest.mark.parametrize( + ("arguments", "expected"), + [ + ("", {}), + (None, {}), + ("null", {}), + ('{"path": "a.rs"', {"path": "a.rs"}), + ('{"path": "a.rs", "limit": 2}', {"path": "a.rs", "limit": 2}), + ({"path": "a.rs"}, {"path": "a.rs"}), + ], + ids=["empty", "null", "null_string", "truncated", "non_string_value", "object"], + ) + def test_replayed_tool_call_arguments_are_forwarded_as_an_object(self, arguments: object, expected: object): + result: Final = OllamaChatConfig().transform_request( + model="qwen3:8b", + messages=self._replayed_history(arguments), + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["messages"][1]["tool_calls"] == [ + {"id": "call_abc123", "function": {"name": "read_file", "arguments": expected}} + ] + + @pytest.mark.parametrize( + "arguments", + ['{"path": "a.rs"}{"path": "b.rs"}', '["a.rs", "b.rs"]', '[["path", "a.rs"]]', "[]"], + ids=["glued", "array", "pairs", "empty_array"], + ) + def test_replayed_tool_call_arguments_that_are_not_an_object_answer_400_without_echoing_them( + self, arguments: str + ): + with pytest.raises(litellm.BadRequestError) as exc_info: + litellm.completion( + model="ollama_chat/qwen3:8b", + messages=self._replayed_history(arguments), + api_base="http://127.0.0.1:9", + ) + + assert exc_info.value.status_code == 400 + assert "call_abc123" in str(exc_info.value) + assert "a.rs" not in "".join(traceback.format_exception(exc_info.value)) + + +class TestOllamaStreamingParallelToolCallIndex: + """Regression for https://github.com/BerriAI/litellm/issues/33678. + + Ollama streams each parallel tool call in its own chunk without an OpenAI-style top-level + index. Every delta used to carry index 0, so index-accumulating clients appended the + arguments of every later call onto the first one. + """ + + @staticmethod + def _tool_call(index: int, command: str) -> dict[str, object]: + return { + "id": f"call_{index}", + "function": {"index": index, "name": "bash", "arguments": {"command": command}}, + } + + @staticmethod + def _chunk(*tool_calls: dict[str, object]) -> dict[str, object]: + return { + "model": "qwen3:8b", + "message": {"role": "assistant", "content": "", "tool_calls": list(tool_calls)}, + "done": False, + } + + @staticmethod + def _parsed_tool_calls( + iterator: OllamaChatCompletionResponseIterator, *chunks: dict[str, object] + ) -> tuple[ChatCompletionDeltaToolCall, ...]: + return tuple( + itertools.chain.from_iterable( + iterator.chunk_parser(chunk).choices[0].delta.tool_calls or () for chunk in chunks + ) + ) + + def test_tool_calls_in_separate_chunks_get_distinct_sequential_indices(self): + iterator: Final = OllamaChatCompletionResponseIterator(streaming_response=iter([]), sync_stream=True) + chunks: Final = [ + self._chunk(self._tool_call(index, command)) + for index, command in enumerate(("date", "uname -a", "whoami")) + ] + + tool_calls: Final = self._parsed_tool_calls(iterator, *chunks) + + assert [tool_call.index for tool_call in tool_calls] == [0, 1, 2] + assert [json.loads(tool_call.function.arguments) for tool_call in tool_calls] == [ + {"command": "date"}, + {"command": "uname -a"}, + {"command": "whoami"}, + ] + + def test_index_counter_continues_across_chunks(self): + iterator: Final = OllamaChatCompletionResponseIterator(streaming_response=iter([]), sync_stream=True) + chunks: Final = [ + self._chunk(self._tool_call(0, "date"), self._tool_call(1, "uname -a")), + self._chunk(self._tool_call(2, "whoami")), + ] + + tool_calls: Final = self._parsed_tool_calls(iterator, *chunks) + + assert [tool_call.index for tool_call in tool_calls] == [0, 1, 2] + class TestOllamaStreamingUsage: @staticmethod