diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 63e6cb2b5da..0e06f99890f 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -3,9 +3,11 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` """ import json -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping, Sequence from typing import Final, Literal, cast, overload +from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 + from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_image_mime_type_from_url, ) @@ -23,10 +25,43 @@ from litellm.types.llms.openai import ( ChatCompletionVideoUrlObject, ) -from ....utils import remove_additional_properties, remove_strict_from_schema from ...openai.chat.gpt_transformation import OpenAIGPTConfig +def _is_str_keyed_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: JSON tool objects have str keys + return isinstance(value, Mapping) + + +def _is_strict(tool: object) -> bool: + function: Final = tool.get("function") if _is_str_keyed_mapping(tool) else None + return _is_str_keyed_mapping(function) and function.get("strict") is True + + +def _str_keyed(value: object) -> Mapping[str, object]: + return value if _is_str_keyed_mapping(value) else {} + + +def _forces_a_tool_call(tool_choice: object) -> bool: + return tool_choice == "required" or _is_str_keyed_mapping(tool_choice) + + +def _with_forwardable_strict(tool: object, keep_strict_false: bool) -> object: + if not _is_str_keyed_mapping(tool): + return tool + function: Final = tool.get("function") + if not _is_str_keyed_mapping(function): + return tool + strict: Final = function.get("strict", True) + if strict is True or (keep_strict_false and strict is False): + return tool + return {**tool, "function": {key: value for key, value in function.items() if key != "strict"}} + + +def _tools_with_forwardable_strict(tools: Sequence[object], tool_choice: object) -> Sequence[object]: + keep_strict_false: Final = not _forces_a_tool_call(tool_choice) and any(_is_strict(tool) for tool in tools) + return [_with_forwardable_strict(tool, keep_strict_false) for tool in tools] + + class HostedVLLMChatConfig(OpenAIGPTConfig): def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, object]]) -> list[dict[str, object]]: """ @@ -86,21 +121,23 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict: - _tools = non_default_params.pop("tools", None) - if _tools is not None: - _tools = remove_additional_properties(_tools) - _tools = remove_strict_from_schema(_tools) - if isinstance(_tools, list): - _tools = self._convert_custom_tools_to_function_tools(_tools) - if _tools is not None: - non_default_params["tools"] = _tools - - thinking: Final = non_default_params.pop("thinking", None) + params: Final = _str_keyed(non_default_params) thinking_effort: Final = ( None - if "reasoning_effort" in non_default_params - else reasoning_effort_from_thinking(thinking, drop_params=drop_params) + if "reasoning_effort" in params + else reasoning_effort_from_thinking(params.get("thinking"), drop_params=drop_params) ) + tools: Final = non_default_params.pop("tools", None) + if tools is not None: + non_default_params["tools"] = ( + _tools_with_forwardable_strict( + self._convert_custom_tools_to_function_tools(tools), params.get("tool_choice") + ) + if isinstance(tools, list) + else tools + ) + + non_default_params.pop("thinking", None) if thinking_effort is not None: non_default_params["reasoning_effort"] = thinking_effort diff --git a/tests/integration/providers/test_hosted_vllm_strict_tools_wire.py b/tests/integration/providers/test_hosted_vllm_strict_tools_wire.py new file mode 100644 index 00000000000..7fdc89360c3 --- /dev/null +++ b/tests/integration/providers/test_hosted_vllm_strict_tools_wire.py @@ -0,0 +1,66 @@ +import uuid +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.openai_wire import answering_model_discovery, chat_reply +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "qwen3-strict-tools" +_API_KEY: Final = "synthetic-hosted-vllm-key" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_TOOLS: Final[list[JsonValue]] = [ + { + "type": "function", + "function": { + "name": "record_code", + "description": "Record a ten letter code", + "strict": True, + "parameters": { + "type": "object", + "properties": { + "code": { + "type": "object", + "properties": {"value": {"type": "string", "pattern": "^Z{10}$"}}, + "required": ["value"], + "additionalProperties": False, + } + }, + "required": ["code"], + "additionalProperties": False, + }, + }, + } +] + + +@pytest.mark.parametrize("drop_params", [True, False]) +def test_hosted_vllm_strict_tool_schema_reaches_the_wire_as_sent(gateway: Gateway, drop_params: bool) -> None: + identity: Final = f"hosted-vllm-strict-tools-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", "/v1/chat/completions"), request + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["tools"] == _TOOLS, body["tools"] + assert body["tool_choice"] == "auto", body + return chat_reply(identity, _BACKEND, "recorded", stream=False) + + with wire_server(answering_model_discovery(respond)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY, drop_params=drop_params + ) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": "Record the code ZZZZZZZZZZ."}], + "tools": _TOOLS, + "tool_choice": "auto", + "num_retries": 0, + "cache": {"no-cache": True}, + }, + ) + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["id"] == identity diff --git a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index c792d5dffcd..4574af98a9c 100644 --- a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -1,14 +1,18 @@ +import copy import json +from collections.abc import Mapping +from typing import Final from unittest.mock import MagicMock, patch import pytest - +from pydantic import BaseModel, ConfigDict from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, ) from litellm.llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig +from litellm.utils import get_optional_params def test_hosted_vllm_chat_transformation_file_url(): @@ -397,3 +401,175 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema(): assert tools[0]["function"]["name"] == "search" assert tools[0]["function"]["description"] == "Search docs" assert tools[0]["function"]["parameters"] == input_schema + + +def _record_code_function() -> Mapping[str, object]: + return { + "name": "record_code", + "description": "Record a ten letter code", + "parameters": { + "type": "object", + "properties": { + "code": { + "type": "object", + "properties": {"value": {"type": "string", "pattern": "^Z{10}$"}}, + "required": ["value"], + "additionalProperties": False, + } + }, + "required": ["code"], + "additionalProperties": False, + }, + } + + +def _strict_tools(strict: object = True) -> list[dict[str, object]]: + return [{"type": "function", "function": {**_record_code_function(), "strict": strict}}] + + +def _unmarked_tool() -> Mapping[str, object]: + return {"type": "function", "function": _record_code_function()} + + +@pytest.mark.parametrize("drop_params", [True, False]) +def test_hosted_vllm_forwards_strict_tool_schema_as_sent(drop_params: bool) -> None: + tools: Final = _strict_tools() + sent: Final = copy.deepcopy(tools) + + optional_params: Final = get_optional_params( + model="my-vllm-model", + custom_llm_provider="hosted_vllm", + tools=tools, + tool_choice="auto", + drop_params=drop_params, + ) + + assert optional_params["tools"] == sent + assert tools == sent + + +@pytest.mark.parametrize("unmarked_strict", [False, None, "true", 1]) +def test_hosted_vllm_drops_a_strict_that_is_not_true(unmarked_strict: object) -> None: + tools: Final = [*_strict_tools(), *_strict_tools(unmarked_strict)] + sent: Final = copy.deepcopy(tools) + + optional_params: Final = get_optional_params( + model="my-vllm-model", + custom_llm_provider="hosted_vllm", + tools=tools, + tool_choice="required", + ) + + assert optional_params["tools"] == [*_strict_tools(), _unmarked_tool()] + assert tools == sent + + +@pytest.mark.parametrize( + ("tool_choice", "keeps_strict_false"), + [ + (None, True), + ("auto", True), + ("required", False), + ({"type": "function", "function": {"name": "record_code"}}, False), + ], +) +def test_hosted_vllm_keeps_strict_false_beside_a_strict_tool_unless_a_call_is_forced( + tool_choice: object, keeps_strict_false: bool +) -> None: + tools: Final = [*_strict_tools(), *_strict_tools(False)] + + optional_params: Final = get_optional_params( + model="my-vllm-model", + custom_llm_provider="hosted_vllm", + tools=tools, + tool_choice=tool_choice, + ) + + expected_second: Final = _strict_tools(False)[0] if keeps_strict_false else _unmarked_tool() + assert optional_params["tools"] == [*_strict_tools(), expected_second] + + +def test_hosted_vllm_drops_strict_false_when_no_tool_is_strict() -> None: + optional_params: Final = get_optional_params( + model="my-vllm-model", + custom_llm_provider="hosted_vllm", + tools=_strict_tools(False), + tool_choice="auto", + ) + + assert optional_params["tools"] == [_unmarked_tool()] + + +class _PydanticTool(BaseModel): + model_config = ConfigDict(frozen=True) + + type: str = "function" + + +@pytest.mark.parametrize("non_dict_tool", [_PydanticTool(), None]) +def test_hosted_vllm_passes_tools_that_are_not_dicts_through(non_dict_tool: object) -> None: + optional_params: Final = get_optional_params( + model="my-vllm-model", + custom_llm_provider="hosted_vllm", + tools=[*_strict_tools(), non_dict_tool], + tool_choice="auto", + ) + + assert optional_params["tools"] == [*_strict_tools(), non_dict_tool] + + +def test_hosted_vllm_additional_drop_params_removes_only_the_named_tool_fields() -> None: + tools: Final = _strict_tools() + sent: Final = copy.deepcopy(tools) + + optional_params: Final = get_optional_params( + model="my-vllm-model", + custom_llm_provider="hosted_vllm", + tools=tools, + tool_choice="auto", + additional_drop_params=["tools[*].function.strict", "tools[*].function.parameters.additionalProperties"], + ) + + assert optional_params["tools"] == [ + { + "type": "function", + "function": { + "name": "record_code", + "description": "Record a ten letter code", + "parameters": { + "type": "object", + "properties": { + "code": { + "type": "object", + "properties": {"value": {"type": "string", "pattern": "^Z{10}$"}}, + "required": ["value"], + "additionalProperties": False, + } + }, + "required": ["code"], + }, + }, + } + ] + assert tools == sent + + +def test_hosted_vllm_additional_drop_params_removes_function_level_additional_properties() -> None: + parameters: Final = {"type": "object", "properties": {"q": {"type": "string"}}, "required": ["q"]} + tools: Final = [ + { + "type": "function", + "function": {"name": "search", "parameters": parameters, "additionalProperties": False, "strict": True}, + } + ] + + optional_params: Final = get_optional_params( + model="my-vllm-model", + custom_llm_provider="hosted_vllm", + tools=tools, + additional_drop_params=["tools[*].function.additionalProperties"], + ) + + assert optional_params["tools"] == [ + {"type": "function", "function": {"name": "search", "parameters": parameters, "strict": True}} + ] diff --git a/tests/unit/test_utils_get_optional_params.py b/tests/unit/test_utils_get_optional_params.py index 05b776cb873..0419c28b75f 100644 --- a/tests/unit/test_utils_get_optional_params.py +++ b/tests/unit/test_utils_get_optional_params.py @@ -685,7 +685,6 @@ def test_unmapped_gemini_model_params(): @pytest.mark.parametrize( "provider, model", [ - ("hosted_vllm", "my-vllm-model"), ("gemini", "gemini-1.5-pro"), ("vertex_ai", "gemini-1.5-pro"), ],