mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(mistral): parse chat.multi_completion responses from hosted tools
This commit is contained in:
parent
658f50663d
commit
83532ebe0a
2 changed files with 223 additions and 1 deletions
|
|
@ -6,7 +6,7 @@ Why separate file? Make it easy to see how transformation works
|
|||
Docs - https://docs.mistral.ai/api/
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncIterator, Coroutine, Iterator
|
||||
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, cast, get_type_hints, overload
|
||||
|
||||
import httpx
|
||||
|
|
@ -447,6 +447,82 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
break
|
||||
return all_expected_values_are_empty
|
||||
|
||||
@staticmethod
|
||||
def _is_multi_completion_choice(choice: object) -> bool:
|
||||
return isinstance(choice, dict) and "messages" in choice and "message" not in choice
|
||||
|
||||
@staticmethod
|
||||
def _multi_completion_content_blocks(turn: Mapping[str, object]) -> Sequence[Mapping[str, object]]:
|
||||
content = turn.get("content")
|
||||
if isinstance(content, str):
|
||||
return ({"type": "text", "text": content},)
|
||||
if isinstance(content, list):
|
||||
return tuple(block for block in content if isinstance(block, dict))
|
||||
return ()
|
||||
|
||||
@staticmethod
|
||||
def _flatten_multi_completion_choice(choice: Mapping[str, object]) -> Mapping[str, object]:
|
||||
raw_turns = choice.get("messages")
|
||||
turns = tuple(turn for turn in raw_turns if isinstance(turn, dict)) if isinstance(raw_turns, list) else ()
|
||||
answered_ids = frozenset(
|
||||
turn.get("tool_call_id") for turn in turns if turn.get("role") == "tool" and turn.get("tool_call_id")
|
||||
)
|
||||
assistant_turns = tuple(turn for turn in turns if turn.get("role") == "assistant")
|
||||
tool_calls = [
|
||||
tool_call
|
||||
for turn in assistant_turns
|
||||
for raw_tool_calls in (turn.get("tool_calls"),)
|
||||
if isinstance(raw_tool_calls, list)
|
||||
for tool_call in raw_tool_calls
|
||||
if isinstance(tool_call, dict) and tool_call.get("id") not in answered_ids
|
||||
]
|
||||
blocks = tuple(
|
||||
block for turn in assistant_turns for block in MistralConfig._multi_completion_content_blocks(turn)
|
||||
)
|
||||
text_parts = [
|
||||
text
|
||||
for block in blocks
|
||||
if block.get("type") == "text" and isinstance(text := block.get("text"), str) and text
|
||||
]
|
||||
image_urls = tuple(
|
||||
url for block in blocks if block.get("type") == "image_url" and (url := block.get("image_url"))
|
||||
)
|
||||
images = [
|
||||
{"type": "image_url", "image_url": url if isinstance(url, dict) else {"url": url}, "index": index}
|
||||
for index, url in enumerate(image_urls)
|
||||
]
|
||||
message = {
|
||||
"role": "assistant",
|
||||
"content": "\n".join(text_parts) if text_parts else None,
|
||||
**({"tool_calls": tool_calls} if tool_calls else {}),
|
||||
**({"images": images} if images else {}),
|
||||
}
|
||||
return {**{key: value for key, value in choice.items() if key != "messages"}, "message": message}
|
||||
|
||||
@staticmethod
|
||||
def _handle_multi_completion_response(response_data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""
|
||||
Mistral's hosted tools on /chat/completions (image_generation, connectors) return
|
||||
``object: "chat.multi_completion"`` where each choice carries ``messages`` (plural,
|
||||
an array of turns) instead of the OpenAI ``message``. Flatten each such choice into
|
||||
a single assistant message: concatenated text content, generated images mapped into
|
||||
``message.images``, and only tool calls left unanswered by a tool turn surfaced as
|
||||
``tool_calls`` (server-executed calls already have their results inlined).
|
||||
"""
|
||||
choices = response_data.get("choices")
|
||||
if not isinstance(choices, list) or not any(MistralConfig._is_multi_completion_choice(c) for c in choices):
|
||||
return response_data
|
||||
return {
|
||||
**response_data,
|
||||
"object": "chat.completion",
|
||||
"choices": [
|
||||
MistralConfig._flatten_multi_completion_choice(choice)
|
||||
if MistralConfig._is_multi_completion_choice(choice)
|
||||
else choice
|
||||
for choice in choices
|
||||
],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _handle_empty_content_response(response_data: dict) -> dict:
|
||||
"""
|
||||
|
|
@ -569,6 +645,7 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
response_data = raw_response.json()
|
||||
response_data = self._handle_empty_content_response(response_data)
|
||||
response_data = self._handle_content_list_to_str_conversion(response_data)
|
||||
response_data = {**self._handle_multi_completion_response(response_data)}
|
||||
|
||||
final_response_obj: Final = cast(
|
||||
ModelResponse,
|
||||
|
|
|
|||
|
|
@ -844,3 +844,148 @@ def test_mistral_transform_request_hoists_tool_message_image():
|
|||
{"type": "text", "text": TOOL_RESULT_IMAGE_BOUNDARY},
|
||||
{"type": "image_url", "image_url": {"url": data_uri}},
|
||||
]
|
||||
|
||||
|
||||
class TestMistralMultiCompletionResponse:
|
||||
"""Regression tests for issue #34228: Mistral hosted tools (image_generation,
|
||||
connectors) return ``object: "chat.multi_completion"`` with plural
|
||||
``choices[].messages``, which used to crash the converter with KeyError: 'message'."""
|
||||
|
||||
@staticmethod
|
||||
def _payload() -> dict:
|
||||
return {
|
||||
"id": "chatcmpl-multi-123",
|
||||
"object": "chat.multi_completion",
|
||||
"created": 1677652288,
|
||||
"model": "mistral-medium-latest",
|
||||
"usage": {"prompt_tokens": 716, "completion_tokens": 355, "total_tokens": 1071, "request_count": 2},
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"messages": [
|
||||
{
|
||||
"role": "assistant",
|
||||
"index": 0,
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "tool-call-1",
|
||||
"type": "function",
|
||||
"function": {"name": "generate_image", "arguments": '{"prompt": "a sunset"}'},
|
||||
"index": 0,
|
||||
"metadata": {"tool_type": "image", "integration_id": "img-1"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"index": 1,
|
||||
"tool_call_id": "tool-call-1",
|
||||
"content": '{"url": "https://cdn.example/image.jpg"}',
|
||||
"metadata": {"tool_call_result": {"type": "success"}},
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"index": 2,
|
||||
"content": [
|
||||
{"type": "text", "text": "Here is your image:"},
|
||||
{"type": "image_url", "image_url": "https://cdn.example/image.jpg"},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
def test_completion_parses_hosted_image_generation_response(self, respx_mock, monkeypatch):
|
||||
import litellm
|
||||
|
||||
monkeypatch.setenv("MISTRAL_API_KEY", "fake-mistral-api-key-12345")
|
||||
respx_mock.post("https://api.mistral.ai/v1/chat/completions").respond(json=self._payload())
|
||||
|
||||
response = litellm.completion(
|
||||
model="mistral/mistral-medium-latest",
|
||||
messages=[{"role": "user", "content": "Generate an image of a sunset over the ocean."}],
|
||||
tools=[{"type": "image_generation"}],
|
||||
)
|
||||
|
||||
message = response.choices[0].message
|
||||
assert message.content == "Here is your image:"
|
||||
assert message.tool_calls is None
|
||||
assert message.images == [
|
||||
{"type": "image_url", "image_url": {"url": "https://cdn.example/image.jpg"}, "index": 0}
|
||||
]
|
||||
assert response.choices[0].finish_reason == "stop"
|
||||
assert response.object == "chat.completion"
|
||||
assert response.usage.prompt_tokens == 716
|
||||
assert response.usage.total_tokens == 1071
|
||||
|
||||
def test_flatten_surfaces_only_unanswered_tool_calls(self):
|
||||
answered = {
|
||||
"id": "answered-1",
|
||||
"type": "function",
|
||||
"function": {"name": "generate_image", "arguments": "{}"},
|
||||
}
|
||||
unanswered = {
|
||||
"id": "pending-1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
}
|
||||
flattened = MistralConfig._handle_multi_completion_response(
|
||||
{
|
||||
"object": "chat.multi_completion",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "tool_calls",
|
||||
"messages": [
|
||||
{"role": "assistant", "content": "", "tool_calls": [answered, unanswered]},
|
||||
{"role": "tool", "tool_call_id": "answered-1", "content": '{"url": "https://x"}'},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
message = flattened["choices"][0]["message"]
|
||||
assert message["tool_calls"] == [unanswered]
|
||||
assert message["content"] is None
|
||||
assert flattened["object"] == "chat.completion"
|
||||
assert "messages" not in flattened["choices"][0]
|
||||
|
||||
def test_flatten_concatenates_assistant_text_across_turns(self):
|
||||
flattened = MistralConfig._handle_multi_completion_response(
|
||||
{
|
||||
"object": "chat.multi_completion",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"messages": [
|
||||
{"role": "assistant", "content": "Searching now."},
|
||||
{"role": "tool", "tool_call_id": "t1", "content": "result"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{"type": "text", "text": "Done."},
|
||||
{"type": "image_url", "image_url": {"url": "https://cdn.example/a.png"}},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
message = flattened["choices"][0]["message"]
|
||||
assert message["content"] == "Searching now.\nDone."
|
||||
assert message["images"] == [
|
||||
{"type": "image_url", "image_url": {"url": "https://cdn.example/a.png"}, "index": 0}
|
||||
]
|
||||
|
||||
def test_standard_chat_completion_response_untouched(self):
|
||||
payload = {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
}
|
||||
assert MistralConfig._handle_multi_completion_response(payload) is payload
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue