From 9e80943c135877eae32426f462c8fa73a84b8057 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 9 Nov 2026 19:49:13 -0800 Subject: [PATCH] fix(hosted_vllm): omit empty tools list from chat requests (#50812) * fix: omit empty tools from chat completions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * Fix empty tools only for hosted vLLM chat completions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Krrish Dholakia Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/hosted_vllm/chat/transformation.py | 20 ++++--- .../test_hosted_vllm_chat_transformation.py | 57 +++++++++++++++++++ 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index c5a0040eb80..e42f4284ce8 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -87,25 +87,31 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict: - _tools = non_default_params.pop("tools", None) + params: Final = dict[str, object](non_default_params) + mapped_params: Final = { + key: value + for key, value in params.items() + if params.get("tools") != [] or key not in ("tools", "tool_choice", "parallel_tool_calls") + } + _tools = mapped_params.pop("tools", None) if _tools is not None: - _tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params + _tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportArgumentType] # tools arrive untyped through the bare-dict params _tools = remove_strict_from_schema(_tools) if isinstance(_tools, list): _tools = self._convert_custom_tools_to_function_tools(_tools) if _tools is not None: - non_default_params["tools"] = _tools + mapped_params["tools"] = _tools - thinking: Final = non_default_params.pop("thinking", None) + thinking: Final = mapped_params.pop("thinking", None) thinking_effort: Final = ( None - if "reasoning_effort" in non_default_params + if "reasoning_effort" in mapped_params else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) ) if thinking_effort is not None: - non_default_params["reasoning_effort"] = thinking_effort + mapped_params["reasoning_effort"] = thinking_effort - return super().map_openai_params(non_default_params, optional_params, model, drop_params) + return super().map_openai_params(mapped_params, optional_params, model, drop_params) def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None diff --git a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index 94f1873faba..3534c7c27e8 100644 --- a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -1,3 +1,4 @@ +import asyncio import copy import json from typing import Final @@ -6,14 +7,70 @@ from unittest.mock import MagicMock, patch import httpx import pytest +import litellm from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, ) from litellm.llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.utils import ModelResponse +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("dependent_options", [False, True]) +@pytest.mark.parametrize("tools_state", ["empty", "absent", "nonempty"]) +@pytest.mark.asyncio +async def test_hosted_vllm_omits_empty_tools_at_http_edge( + asynchronous: bool, dependent_options: bool, tools_state: str +) -> None: + requests: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + tools: Final = [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}] + messages: Final = [ + {"role": "user", "content": "Look up x"}, + {"role": "assistant", "tool_calls": [ + {"id": "call_123", "type": "function", "function": {"name": "lookup", "arguments": "{}"}} + ]}, + {"role": "tool", "tool_call_id": "call_123", "content": "OK"}, + {"role": "user", "content": "Say OK"}, + ] + options: Final = { + **({} if tools_state == "absent" else {"tools": [] if tools_state == "empty" else tools}), + **({"tool_choice": "required", "parallel_tool_calls": False} if dependent_options else {}), + } + + def respond(request: httpx.Request) -> httpx.Response: + requests.put_nowait(request) + return httpx.Response(200, json={ + "id": "chatcmpl-test", "object": "chat.completion", "created": 1, "model": "test", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "OK"}, "finish_reason": "stop"}], + }) + + if asynchronous: + async_client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + try: + await litellm.acompletion( + model="hosted_vllm/test", api_base="https://upstream.test/v1", api_key="test-key", + messages=messages, client=async_client, **options, + ) + finally: + await async_client.close() + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as sync_client: + litellm.completion( + model="hosted_vllm/test", api_base="https://upstream.test/v1", api_key="test-key", + messages=messages, client=HTTPHandler(client=sync_client), **options, + ) + sent: Final = requests.get_nowait() + assert str(sent.url) == "https://upstream.test/v1/chat/completions" + assert json.loads(sent.content) == { + "model": "test", "messages": messages, + **({"tools": tools} if tools_state == "nonempty" else {}), + **({"tool_choice": "required"} if dependent_options and tools_state == "nonempty" else {}), + **({"parallel_tool_calls": False} if dependent_options and tools_state != "empty" else {}), + } + + def test_hosted_vllm_chat_transformation_file_url(): config = HostedVLLMChatConfig() video_url = "https://example.com/video.mp4"