diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index de896cde4dc..5b9c9a15197 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1131,48 +1131,67 @@ def anthropic_messages_pt_xml(messages: list): def _azure_tool_call_invoke_helper( function_call_params: ChatCompletionToolCallFunctionChunk, -) -> ChatCompletionToolCallFunctionChunk | None: +) -> ChatCompletionToolCallFunctionChunk: """ Azure requires 'arguments' to be a string. """ - if function_call_params.get("arguments") is None: - function_call_params["arguments"] = "" - return function_call_params + if function_call_params.get("arguments") is not None: + return function_call_params + return {**function_call_params, "arguments": ""} -def _azure_image_url_helper(content: ChatCompletionImageObject): - if isinstance(content["image_url"], str): - content["image_url"] = {"url": content["image_url"]} - else: - content["image_url"] = cast( - ChatCompletionImageUrlObject, - {k: v for k, v in content["image_url"].items() if k != "format"}, - ) +def _azure_image_url_helper(content: ChatCompletionImageObject) -> ChatCompletionImageObject: + image_url: Final = content["image_url"] + if isinstance(image_url, str): + return {**content, "image_url": {"url": image_url}} + return { + **content, + "image_url": cast( # cast-ok: dropping format keeps the required url + ChatCompletionImageUrlObject, {k: v for k, v in image_url.items() if k != "format"} + ), + } -def _azure_file_helper(content: ChatCompletionFileObject) -> None: - content["file"] = cast( - ChatCompletionFileObjectFile, - {k: v for k, v in content.get("file", {}).items() if k != "format"}, +def _azure_file_helper(content: ChatCompletionFileObject) -> ChatCompletionFileObject: + return { + **content, + "file": cast( # cast-ok: dropping format leaves only ChatCompletionFileObjectFile keys + ChatCompletionFileObjectFile, {k: v for k, v in content.get("file", {}).items() if k != "format"} + ), + } + + +def _azure_content_part(part: object) -> object: + mapping: Final = as_str_mapping(part) + part_type: Final = None if mapping is None else mapping.get("type") + if part_type == "image_url": + return _azure_image_url_helper(cast(ChatCompletionImageObject, part)) # cast-ok: the type tag says image_url + if part_type == "file": + return _azure_file_helper(cast(ChatCompletionFileObject, part)) # cast-ok: the type tag says file + return part + + +def _azure_message(message: AllMessageValues) -> AllMessageValues: + if message["role"] == "assistant": + function_call: Final = message.get("function_call") + if function_call is None: + return message + return {**message, "function_call": _azure_tool_call_invoke_helper(function_call)} + if message["role"] != "user": + return message + content: Final = message.get("content") + if not isinstance(content, list): + return message + parts: Final[tuple[object, ...]] = tuple(content) + return cast( # cast-ok: only image_url and file parts change, each to the shape Azure takes + AllMessageValues, {**message, "content": [_azure_content_part(part) for part in parts]} ) def convert_to_azure_openai_messages( messages: list[AllMessageValues], ) -> list[AllMessageValues]: - for m in messages: - if m["role"] == "assistant": - function_call = m.get("function_call", None) - if function_call is not None: - m["function_call"] = _azure_tool_call_invoke_helper(function_call) - - if m["role"] == "user" and isinstance(m.get("content"), list): - for content in m.get("content", []): - if isinstance(content, dict) and content.get("type") == "image_url": - _azure_image_url_helper(cast(ChatCompletionImageObject, content)) - elif isinstance(content, dict) and content.get("type") == "file": - _azure_file_helper(cast(ChatCompletionFileObject, content)) - return messages + return [_azure_message(message) for message in messages] # ------------------------------------------------------------------------------ diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 9f9503f93b6..8e27458da2a 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -3,9 +3,10 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` """ import json -from collections.abc import Coroutine +from collections.abc import Coroutine, Iterable, Mapping, Sequence from typing import Final, Literal, cast, overload +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_image_mime_type_from_url, ) @@ -16,16 +17,89 @@ from litellm.litellm_core_utils.reasoning_effort_utils import ( from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, + ChatCompletionAssistantMessage, ChatCompletionAssistantToolCall, ChatCompletionFileObject, - ChatCompletionToolCallFunctionChunk, + ChatCompletionUserMessage, ChatCompletionVideoObject, ChatCompletionVideoUrlObject, + OpenAIMessageContentListBlock, ) from ....utils import remove_additional_properties, remove_strict_from_schema from ...openai.chat.gpt_transformation import OpenAIGPTConfig +_THINKING_KEYS: Final = frozenset({"thinking_blocks"}) +_THINKING_AND_REASONING_KEYS: Final = frozenset({"thinking_blocks", "reasoning_content"}) + + +def _content_part_type(part: object) -> object: + mapping: Final = as_str_mapping(part) + return None if mapping is None else mapping.get("type") + + +def _tool_use_as_tool_call(part: Mapping[str, object]) -> ChatCompletionAssistantToolCall: + tool_input: Final = part.get("input", {}) + return cast( # cast-ok: id and name are forwarded as the caller sent them + ChatCompletionAssistantToolCall, + { + "id": part.get("id"), + "type": "function", + "function": { + "name": part.get("name"), + "arguments": tool_input if isinstance(tool_input, str) else json.dumps(tool_input), + }, + }, + ) + + +def _tool_call_ids(tool_calls: Iterable[object]) -> frozenset[object]: + mappings: Final = tuple(mapping for mapping in map(as_str_mapping, tool_calls) if mapping is not None) + return frozenset(mapping.get("id") for mapping in mappings if mapping.get("id") is not None) + + +def _tool_calls_update( + existing: Sequence[ChatCompletionAssistantToolCall] | None, + from_content: tuple[ChatCompletionAssistantToolCall, ...], +) -> Mapping[str, Sequence[ChatCompletionAssistantToolCall]]: + if not from_content: + return {} + if not isinstance(existing, list): + return {"tool_calls": list(from_content)} + existing_ids: Final = _tool_call_ids(existing) + new_tool_calls: Final = tuple(tool_call for tool_call in from_content if tool_call["id"] not in existing_ids) + return {"tool_calls": [*existing, *new_tool_calls]} if new_tool_calls else {} + + +def _vllm_assistant_message(message: ChatCompletionAssistantMessage) -> AllMessageValues: + dropped_keys: Final = ( + _THINKING_KEYS if isinstance(message.get("reasoning_content"), str) else _THINKING_AND_REASONING_KEYS + ) + kept: Final = {key: value for key, value in message.items() if key not in dropped_keys} + content: Final = message.get("content") + if not isinstance(content, list): + return cast(AllMessageValues, kept) # cast-ok: only keys vLLM rejects are removed + parts: Final[tuple[object, ...]] = tuple(content) + mappings: Final = tuple(mapping for mapping in map(as_str_mapping, parts) if mapping is not None) + texts: Final = tuple( + cast(str, mapping.get("text", "")) # cast-ok: a text part carries str text, and join rejects anything else + for mapping in mappings + if mapping.get("type") == "text" + ) + tool_calls: Final = tuple( + _tool_use_as_tool_call(mapping) for mapping in mappings if mapping.get("type") == "tool_use" + ) + kept_parts: Final = [part for part in parts if _content_part_type(part) != "tool_use"] + has_structured_content: Final = any(_content_part_type(part) not in ("text", "tool_use") for part in parts) + return cast( # cast-ok: content and tool_calls keep the shapes vLLM takes + AllMessageValues, + { + **kept, + "content": kept_parts if has_structured_content else "\n".join(texts), + **_tool_calls_update(message.get("tool_calls"), tool_calls), + }, + ) + class HostedVLLMChatConfig(OpenAIGPTConfig): def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, object]]) -> list[dict[str, object]]: @@ -172,76 +246,28 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): - thinking_blocks and non-string reasoning_content on assistant messages are removed, and content lists are converted to strings for vLLM compatibility """ - for message in messages: - if message["role"] == "assistant": - message.pop("thinking_blocks", None) - if not isinstance(message.get("reasoning_content"), str): - message.pop("reasoning_content", None) - existing_content = message.get("content") - if isinstance(existing_content, list): - text_parts = [] - tool_calls: list[ChatCompletionAssistantToolCall] = [] - content_blocks: list[object] = [] - has_structured_content = False - for c in existing_content: - if isinstance(c, dict) and c.get("type") == "text": - text_parts.append(c.get("text", "")) - content_blocks.append(c) - elif isinstance(c, dict) and c.get("type") == "tool_use": - tool_input = c.get("input", {}) - tool_calls.append( - ChatCompletionAssistantToolCall( - id=c.get("id"), - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=c.get("name"), - arguments=( - tool_input - if isinstance( - tool_input, - str, - ) - else json.dumps(tool_input) - ), - ), - ) - ) - else: - content_blocks.append(c) - has_structured_content = True - if tool_calls: - existing_tool_calls = message.get("tool_calls") - if isinstance(existing_tool_calls, list): - existing_tool_call_ids = { - tool_call.get("id") - for tool_call in existing_tool_calls - if isinstance(tool_call, dict) and tool_call.get("id") is not None - } - new_tool_calls = [ - tool_call - for tool_call in tool_calls - if tool_call.get("id") not in existing_tool_call_ids - ] - if new_tool_calls: - message["tool_calls"] = existing_tool_calls + new_tool_calls - else: - message["tool_calls"] = tool_calls - content_str = "\n".join(text_parts) - new_content = content_blocks if has_structured_content else content_str - message["content"] = new_content - elif message["role"] == "user": - message_content = message.get("content") - if message_content and isinstance(message_content, list): - replaced_content_items: list[tuple[int, ChatCompletionFileObject]] = [] - for idx, content_item in enumerate(message_content): - if content_item.get("type") == "file": - content_item = cast(ChatCompletionFileObject, content_item) - if self._is_video_file(content_item): - replaced_content_items.append((idx, content_item)) - for idx, content_item in replaced_content_items: - message_content[idx] = self._convert_file_to_video_url(content_item) - + vllm_messages: Final = [self._vllm_message(message) for message in messages] if is_async: - return super()._transform_messages(messages, model, is_async=cast(Literal[True], True)) - else: - return super()._transform_messages(messages, model, is_async=cast(Literal[False], False)) + return super()._transform_messages(vllm_messages, model, is_async=True) + return super()._transform_messages(vllm_messages, model, is_async=False) + + def _vllm_message(self, message: AllMessageValues) -> AllMessageValues: + if message["role"] == "assistant": + return _vllm_assistant_message(message) + if message["role"] == "user": + return self._vllm_user_message(message) + return message + + def _vllm_user_message(self, message: ChatCompletionUserMessage) -> AllMessageValues: + content: Final = message.get("content") + if not content or not isinstance(content, list): + return message + return cast( # cast-ok: only video file parts change, to the video_url part vLLM takes + AllMessageValues, {**message, "content": [self._with_video_url(part) for part in content]} + ) + + def _with_video_url(self, part: OpenAIMessageContentListBlock) -> OpenAIMessageContentListBlock: + if part.get("type") != "file": + return part + file_part: Final = cast(ChatCompletionFileObject, part) # cast-ok: the type tag says this is a file part + return self._convert_file_to_video_url(file_part) if self._is_video_file(file_part) else part diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index f6dae2c304f..9cc3223ff3e 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -1,4 +1,4 @@ -import asyncio, httpx, importlib, json, os +import asyncio, copy, httpx, importlib, json, os import sys from typing import Final @@ -140,6 +140,53 @@ def test_transform_request_drops_tool_reference_parts(): assert request["messages"][2]["content"] == "" +def test_transform_request_leaves_the_caller_messages_untouched(): + city_arguments = json.dumps({"city": "Copenhagen"}) + messages = [ + {"role": "user", "content": "What is the weather where I am?"}, + {"role": "assistant", "content": None, "function_call": {"name": "get_current_city", "arguments": None}}, + {"role": "function", "name": "get_current_city", "content": "Copenhagen"}, + {"role": "assistant", "content": None, "function_call": {"name": "get_weather", "arguments": city_arguments}}, + {"role": "function", "name": "get_weather", "content": "Sunny"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Compare these with the attached report"}, + {"type": "image_url", "image_url": "https://example.com/forecast.png"}, + {"type": "image_url", "image_url": {"url": "https://example.com/radar.png", "format": "image/png"}}, + {"type": "file", "file": {"file_id": "assistant-report", "format": "application/pdf"}}, + ], + }, + ] + snapshot = copy.deepcopy(messages) + + request = AzureOpenAIConfig().transform_request( + model="gpt-4o", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == [ + {"role": "user", "content": "What is the weather where I am?"}, + {"role": "assistant", "content": None, "function_call": {"name": "get_current_city", "arguments": ""}}, + {"role": "function", "name": "get_current_city", "content": "Copenhagen"}, + {"role": "assistant", "content": None, "function_call": {"name": "get_weather", "arguments": city_arguments}}, + {"role": "function", "name": "get_weather", "content": "Sunny"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Compare these with the attached report"}, + {"type": "image_url", "image_url": {"url": "https://example.com/forecast.png"}}, + {"type": "image_url", "image_url": {"url": "https://example.com/radar.png"}}, + {"type": "file", "file": {"file_id": "assistant-report"}}, + ], + }, + ] + assert messages == snapshot + + @pytest.mark.parametrize( "enabled, expected", [(False, ("hi", "sys", "reply", "more")), (True, ("sys", "hi", "reply", "more"))] ) diff --git a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index c792d5dffcd..88f3857b5b4 100644 --- a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -1,3 +1,5 @@ +import asyncio +import copy import json from unittest.mock import MagicMock, patch @@ -337,6 +339,70 @@ def test_hosted_vllm_assistant_tool_use_does_not_duplicate_existing_tool_calls() ] +def _transform_request_sync(config, messages): + return config.transform_request( + model="hosted_vllm/qwen3-vl", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + +def _transform_request_async(config, messages): + return asyncio.run( + config.async_transform_request( + model="hosted_vllm/qwen3-vl", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + ) + + +@pytest.mark.parametrize("transform_request", [_transform_request_sync, _transform_request_async], ids=["sync", "async"]) +def test_hosted_vllm_transform_request_leaves_the_caller_messages_untouched(transform_request): + video_url = "https://example.com/clip.mp4" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Where was this clip filmed, and what is the weather there?"}, + {"type": "file", "file": {"file_id": video_url, "format": "video/mp4"}}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "It was filmed in Boston. Checking the weather."}, + {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Boston"}}, + ], + "thinking_blocks": [{"type": "thinking", "thinking": "The skyline is Boston.", "signature": "sig1"}], + "reasoning_content": [{"type": "text", "text": "The skyline is Boston."}], + }, + {"role": "tool", "tool_call_id": "toolu_1", "content": "Sunny, 21C"}, + ] + snapshot = copy.deepcopy(messages) + + transformed = transform_request(HostedVLLMChatConfig(), messages) + + assert transformed["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Where was this clip filmed, and what is the weather there?"}, + {"type": "video_url", "video_url": {"url": video_url}}, + ], + }, + { + "role": "assistant", + "content": "It was filmed in Boston. Checking the weather.", + "tool_calls": [ + { + "id": "toolu_1", + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": "Boston"})}, + } + ], + }, + {"role": "tool", "tool_call_id": "toolu_1", "content": "Sunny, 21C"}, + ] + assert messages == snapshot + + def test_hosted_vllm_custom_tools_are_converted_to_function_tools(): config = HostedVLLMChatConfig() optional_params = config.map_openai_params(