mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(hosted_vllm,azure): leave the caller's messages untouched when building the chat request
hosted_vllm and Azure rewrote the caller's nested message objects while building the request, so a Router fallback reused the rewritten parts. A video file part reached a Gemini fallback as video_url, which Gemini drops. Both now build new message dicts and content parts and send the same request body as before
This commit is contained in:
mateo-berri
parent
1ccb970d52
commit
db6d0d85e6
4 changed files with 262 additions and 104 deletions
|
|
@ -1131,48 +1131,67 @@ def anthropic_messages_pt_xml(messages: list):
|
|||
|
||||
def _azure_tool_call_invoke_helper(
|
||||
function_call_params: ChatCompletionToolCallFunctionChunk,
|
||||
) -> ChatCompletionToolCallFunctionChunk | None:
|
||||
) -> ChatCompletionToolCallFunctionChunk:
|
||||
"""
|
||||
Azure requires 'arguments' to be a string.
|
||||
"""
|
||||
if function_call_params.get("arguments") is None:
|
||||
function_call_params["arguments"] = ""
|
||||
return function_call_params
|
||||
if function_call_params.get("arguments") is not None:
|
||||
return function_call_params
|
||||
return {**function_call_params, "arguments": ""}
|
||||
|
||||
|
||||
def _azure_image_url_helper(content: ChatCompletionImageObject):
|
||||
if isinstance(content["image_url"], str):
|
||||
content["image_url"] = {"url": content["image_url"]}
|
||||
else:
|
||||
content["image_url"] = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in content["image_url"].items() if k != "format"},
|
||||
)
|
||||
def _azure_image_url_helper(content: ChatCompletionImageObject) -> ChatCompletionImageObject:
|
||||
image_url: Final = content["image_url"]
|
||||
if isinstance(image_url, str):
|
||||
return {**content, "image_url": {"url": image_url}}
|
||||
return {
|
||||
**content,
|
||||
"image_url": cast( # cast-ok: dropping format keeps the required url
|
||||
ChatCompletionImageUrlObject, {k: v for k, v in image_url.items() if k != "format"}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _azure_file_helper(content: ChatCompletionFileObject) -> None:
|
||||
content["file"] = cast(
|
||||
ChatCompletionFileObjectFile,
|
||||
{k: v for k, v in content.get("file", {}).items() if k != "format"},
|
||||
def _azure_file_helper(content: ChatCompletionFileObject) -> ChatCompletionFileObject:
|
||||
return {
|
||||
**content,
|
||||
"file": cast( # cast-ok: dropping format leaves only ChatCompletionFileObjectFile keys
|
||||
ChatCompletionFileObjectFile, {k: v for k, v in content.get("file", {}).items() if k != "format"}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _azure_content_part(part: object) -> object:
|
||||
mapping: Final = as_str_mapping(part)
|
||||
part_type: Final = None if mapping is None else mapping.get("type")
|
||||
if part_type == "image_url":
|
||||
return _azure_image_url_helper(cast(ChatCompletionImageObject, part)) # cast-ok: the type tag says image_url
|
||||
if part_type == "file":
|
||||
return _azure_file_helper(cast(ChatCompletionFileObject, part)) # cast-ok: the type tag says file
|
||||
return part
|
||||
|
||||
|
||||
def _azure_message(message: AllMessageValues) -> AllMessageValues:
|
||||
if message["role"] == "assistant":
|
||||
function_call: Final = message.get("function_call")
|
||||
if function_call is None:
|
||||
return message
|
||||
return {**message, "function_call": _azure_tool_call_invoke_helper(function_call)}
|
||||
if message["role"] != "user":
|
||||
return message
|
||||
content: Final = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
return message
|
||||
parts: Final[tuple[object, ...]] = tuple(content)
|
||||
return cast( # cast-ok: only image_url and file parts change, each to the shape Azure takes
|
||||
AllMessageValues, {**message, "content": [_azure_content_part(part) for part in parts]}
|
||||
)
|
||||
|
||||
|
||||
def convert_to_azure_openai_messages(
|
||||
messages: list[AllMessageValues],
|
||||
) -> list[AllMessageValues]:
|
||||
for m in messages:
|
||||
if m["role"] == "assistant":
|
||||
function_call = m.get("function_call", None)
|
||||
if function_call is not None:
|
||||
m["function_call"] = _azure_tool_call_invoke_helper(function_call)
|
||||
|
||||
if m["role"] == "user" and isinstance(m.get("content"), list):
|
||||
for content in m.get("content", []):
|
||||
if isinstance(content, dict) and content.get("type") == "image_url":
|
||||
_azure_image_url_helper(cast(ChatCompletionImageObject, content))
|
||||
elif isinstance(content, dict) and content.get("type") == "file":
|
||||
_azure_file_helper(cast(ChatCompletionFileObject, content))
|
||||
return messages
|
||||
return [_azure_message(message) for message in messages]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -3,9 +3,10 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Coroutine
|
||||
from collections.abc import Coroutine, Iterable, Mapping, Sequence
|
||||
from typing import Final, Literal, cast, overload
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_image_mime_type_from_url,
|
||||
)
|
||||
|
|
@ -16,16 +17,89 @@ from litellm.litellm_core_utils.reasoning_effort_utils import (
|
|||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantMessage,
|
||||
ChatCompletionAssistantToolCall,
|
||||
ChatCompletionFileObject,
|
||||
ChatCompletionToolCallFunctionChunk,
|
||||
ChatCompletionUserMessage,
|
||||
ChatCompletionVideoObject,
|
||||
ChatCompletionVideoUrlObject,
|
||||
OpenAIMessageContentListBlock,
|
||||
)
|
||||
|
||||
from ....utils import remove_additional_properties, remove_strict_from_schema
|
||||
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
_THINKING_KEYS: Final = frozenset({"thinking_blocks"})
|
||||
_THINKING_AND_REASONING_KEYS: Final = frozenset({"thinking_blocks", "reasoning_content"})
|
||||
|
||||
|
||||
def _content_part_type(part: object) -> object:
|
||||
mapping: Final = as_str_mapping(part)
|
||||
return None if mapping is None else mapping.get("type")
|
||||
|
||||
|
||||
def _tool_use_as_tool_call(part: Mapping[str, object]) -> ChatCompletionAssistantToolCall:
|
||||
tool_input: Final = part.get("input", {})
|
||||
return cast( # cast-ok: id and name are forwarded as the caller sent them
|
||||
ChatCompletionAssistantToolCall,
|
||||
{
|
||||
"id": part.get("id"),
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": part.get("name"),
|
||||
"arguments": tool_input if isinstance(tool_input, str) else json.dumps(tool_input),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _tool_call_ids(tool_calls: Iterable[object]) -> frozenset[object]:
|
||||
mappings: Final = tuple(mapping for mapping in map(as_str_mapping, tool_calls) if mapping is not None)
|
||||
return frozenset(mapping.get("id") for mapping in mappings if mapping.get("id") is not None)
|
||||
|
||||
|
||||
def _tool_calls_update(
|
||||
existing: Sequence[ChatCompletionAssistantToolCall] | None,
|
||||
from_content: tuple[ChatCompletionAssistantToolCall, ...],
|
||||
) -> Mapping[str, Sequence[ChatCompletionAssistantToolCall]]:
|
||||
if not from_content:
|
||||
return {}
|
||||
if not isinstance(existing, list):
|
||||
return {"tool_calls": list(from_content)}
|
||||
existing_ids: Final = _tool_call_ids(existing)
|
||||
new_tool_calls: Final = tuple(tool_call for tool_call in from_content if tool_call["id"] not in existing_ids)
|
||||
return {"tool_calls": [*existing, *new_tool_calls]} if new_tool_calls else {}
|
||||
|
||||
|
||||
def _vllm_assistant_message(message: ChatCompletionAssistantMessage) -> AllMessageValues:
|
||||
dropped_keys: Final = (
|
||||
_THINKING_KEYS if isinstance(message.get("reasoning_content"), str) else _THINKING_AND_REASONING_KEYS
|
||||
)
|
||||
kept: Final = {key: value for key, value in message.items() if key not in dropped_keys}
|
||||
content: Final = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
return cast(AllMessageValues, kept) # cast-ok: only keys vLLM rejects are removed
|
||||
parts: Final[tuple[object, ...]] = tuple(content)
|
||||
mappings: Final = tuple(mapping for mapping in map(as_str_mapping, parts) if mapping is not None)
|
||||
texts: Final = tuple(
|
||||
cast(str, mapping.get("text", "")) # cast-ok: a text part carries str text, and join rejects anything else
|
||||
for mapping in mappings
|
||||
if mapping.get("type") == "text"
|
||||
)
|
||||
tool_calls: Final = tuple(
|
||||
_tool_use_as_tool_call(mapping) for mapping in mappings if mapping.get("type") == "tool_use"
|
||||
)
|
||||
kept_parts: Final = [part for part in parts if _content_part_type(part) != "tool_use"]
|
||||
has_structured_content: Final = any(_content_part_type(part) not in ("text", "tool_use") for part in parts)
|
||||
return cast( # cast-ok: content and tool_calls keep the shapes vLLM takes
|
||||
AllMessageValues,
|
||||
{
|
||||
**kept,
|
||||
"content": kept_parts if has_structured_content else "\n".join(texts),
|
||||
**_tool_calls_update(message.get("tool_calls"), tool_calls),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class HostedVLLMChatConfig(OpenAIGPTConfig):
|
||||
def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, object]]) -> list[dict[str, object]]:
|
||||
|
|
@ -172,76 +246,28 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
|
|||
- thinking_blocks and non-string reasoning_content on assistant messages
|
||||
are removed, and content lists are converted to strings for vLLM compatibility
|
||||
"""
|
||||
for message in messages:
|
||||
if message["role"] == "assistant":
|
||||
message.pop("thinking_blocks", None)
|
||||
if not isinstance(message.get("reasoning_content"), str):
|
||||
message.pop("reasoning_content", None)
|
||||
existing_content = message.get("content")
|
||||
if isinstance(existing_content, list):
|
||||
text_parts = []
|
||||
tool_calls: list[ChatCompletionAssistantToolCall] = []
|
||||
content_blocks: list[object] = []
|
||||
has_structured_content = False
|
||||
for c in existing_content:
|
||||
if isinstance(c, dict) and c.get("type") == "text":
|
||||
text_parts.append(c.get("text", ""))
|
||||
content_blocks.append(c)
|
||||
elif isinstance(c, dict) and c.get("type") == "tool_use":
|
||||
tool_input = c.get("input", {})
|
||||
tool_calls.append(
|
||||
ChatCompletionAssistantToolCall(
|
||||
id=c.get("id"),
|
||||
type="function",
|
||||
function=ChatCompletionToolCallFunctionChunk(
|
||||
name=c.get("name"),
|
||||
arguments=(
|
||||
tool_input
|
||||
if isinstance(
|
||||
tool_input,
|
||||
str,
|
||||
)
|
||||
else json.dumps(tool_input)
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
else:
|
||||
content_blocks.append(c)
|
||||
has_structured_content = True
|
||||
if tool_calls:
|
||||
existing_tool_calls = message.get("tool_calls")
|
||||
if isinstance(existing_tool_calls, list):
|
||||
existing_tool_call_ids = {
|
||||
tool_call.get("id")
|
||||
for tool_call in existing_tool_calls
|
||||
if isinstance(tool_call, dict) and tool_call.get("id") is not None
|
||||
}
|
||||
new_tool_calls = [
|
||||
tool_call
|
||||
for tool_call in tool_calls
|
||||
if tool_call.get("id") not in existing_tool_call_ids
|
||||
]
|
||||
if new_tool_calls:
|
||||
message["tool_calls"] = existing_tool_calls + new_tool_calls
|
||||
else:
|
||||
message["tool_calls"] = tool_calls
|
||||
content_str = "\n".join(text_parts)
|
||||
new_content = content_blocks if has_structured_content else content_str
|
||||
message["content"] = new_content
|
||||
elif message["role"] == "user":
|
||||
message_content = message.get("content")
|
||||
if message_content and isinstance(message_content, list):
|
||||
replaced_content_items: list[tuple[int, ChatCompletionFileObject]] = []
|
||||
for idx, content_item in enumerate(message_content):
|
||||
if content_item.get("type") == "file":
|
||||
content_item = cast(ChatCompletionFileObject, content_item)
|
||||
if self._is_video_file(content_item):
|
||||
replaced_content_items.append((idx, content_item))
|
||||
for idx, content_item in replaced_content_items:
|
||||
message_content[idx] = self._convert_file_to_video_url(content_item)
|
||||
|
||||
vllm_messages: Final = [self._vllm_message(message) for message in messages]
|
||||
if is_async:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
|
||||
else:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
|
||||
return super()._transform_messages(vllm_messages, model, is_async=True)
|
||||
return super()._transform_messages(vllm_messages, model, is_async=False)
|
||||
|
||||
def _vllm_message(self, message: AllMessageValues) -> AllMessageValues:
|
||||
if message["role"] == "assistant":
|
||||
return _vllm_assistant_message(message)
|
||||
if message["role"] == "user":
|
||||
return self._vllm_user_message(message)
|
||||
return message
|
||||
|
||||
def _vllm_user_message(self, message: ChatCompletionUserMessage) -> AllMessageValues:
|
||||
content: Final = message.get("content")
|
||||
if not content or not isinstance(content, list):
|
||||
return message
|
||||
return cast( # cast-ok: only video file parts change, to the video_url part vLLM takes
|
||||
AllMessageValues, {**message, "content": [self._with_video_url(part) for part in content]}
|
||||
)
|
||||
|
||||
def _with_video_url(self, part: OpenAIMessageContentListBlock) -> OpenAIMessageContentListBlock:
|
||||
if part.get("type") != "file":
|
||||
return part
|
||||
file_part: Final = cast(ChatCompletionFileObject, part) # cast-ok: the type tag says this is a file part
|
||||
return self._convert_file_to_video_url(file_part) if self._is_video_file(file_part) else part
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import asyncio, httpx, importlib, json, os
|
||||
import asyncio, copy, httpx, importlib, json, os
|
||||
import sys
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -140,6 +140,53 @@ def test_transform_request_drops_tool_reference_parts():
|
|||
assert request["messages"][2]["content"] == ""
|
||||
|
||||
|
||||
def test_transform_request_leaves_the_caller_messages_untouched():
|
||||
city_arguments = json.dumps({"city": "Copenhagen"})
|
||||
messages = [
|
||||
{"role": "user", "content": "What is the weather where I am?"},
|
||||
{"role": "assistant", "content": None, "function_call": {"name": "get_current_city", "arguments": None}},
|
||||
{"role": "function", "name": "get_current_city", "content": "Copenhagen"},
|
||||
{"role": "assistant", "content": None, "function_call": {"name": "get_weather", "arguments": city_arguments}},
|
||||
{"role": "function", "name": "get_weather", "content": "Sunny"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Compare these with the attached report"},
|
||||
{"type": "image_url", "image_url": "https://example.com/forecast.png"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/radar.png", "format": "image/png"}},
|
||||
{"type": "file", "file": {"file_id": "assistant-report", "format": "application/pdf"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
snapshot = copy.deepcopy(messages)
|
||||
|
||||
request = AzureOpenAIConfig().transform_request(
|
||||
model="gpt-4o",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["messages"] == [
|
||||
{"role": "user", "content": "What is the weather where I am?"},
|
||||
{"role": "assistant", "content": None, "function_call": {"name": "get_current_city", "arguments": ""}},
|
||||
{"role": "function", "name": "get_current_city", "content": "Copenhagen"},
|
||||
{"role": "assistant", "content": None, "function_call": {"name": "get_weather", "arguments": city_arguments}},
|
||||
{"role": "function", "name": "get_weather", "content": "Sunny"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Compare these with the attached report"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/forecast.png"}},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/radar.png"}},
|
||||
{"type": "file", "file": {"file_id": "assistant-report"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
assert messages == snapshot
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enabled, expected", [(False, ("hi", "sys", "reply", "more")), (True, ("sys", "hi", "reply", "more"))]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
|
@ -337,6 +339,70 @@ def test_hosted_vllm_assistant_tool_use_does_not_duplicate_existing_tool_calls()
|
|||
]
|
||||
|
||||
|
||||
def _transform_request_sync(config, messages):
|
||||
return config.transform_request(
|
||||
model="hosted_vllm/qwen3-vl", messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
|
||||
def _transform_request_async(config, messages):
|
||||
return asyncio.run(
|
||||
config.async_transform_request(
|
||||
model="hosted_vllm/qwen3-vl", messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("transform_request", [_transform_request_sync, _transform_request_async], ids=["sync", "async"])
|
||||
def test_hosted_vllm_transform_request_leaves_the_caller_messages_untouched(transform_request):
|
||||
video_url = "https://example.com/clip.mp4"
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Where was this clip filmed, and what is the weather there?"},
|
||||
{"type": "file", "file": {"file_id": video_url, "format": "video/mp4"}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "It was filmed in Boston. Checking the weather."},
|
||||
{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Boston"}},
|
||||
],
|
||||
"thinking_blocks": [{"type": "thinking", "thinking": "The skyline is Boston.", "signature": "sig1"}],
|
||||
"reasoning_content": [{"type": "text", "text": "The skyline is Boston."}],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "toolu_1", "content": "Sunny, 21C"},
|
||||
]
|
||||
snapshot = copy.deepcopy(messages)
|
||||
|
||||
transformed = transform_request(HostedVLLMChatConfig(), messages)
|
||||
|
||||
assert transformed["messages"] == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Where was this clip filmed, and what is the weather there?"},
|
||||
{"type": "video_url", "video_url": {"url": video_url}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "It was filmed in Boston. Checking the weather.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "toolu_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": json.dumps({"city": "Boston"})},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "toolu_1", "content": "Sunny, 21C"},
|
||||
]
|
||||
assert messages == snapshot
|
||||
|
||||
|
||||
def test_hosted_vllm_custom_tools_are_converted_to_function_tools():
|
||||
config = HostedVLLMChatConfig()
|
||||
optional_params = config.map_openai_params(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue