mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-11 03:38:38 +00:00
fix(deepseek): forward image content in role=tool messages
DeepSeekChatConfig._is_vision_forwardable_content() rejected every non-user role, so a role=tool message carrying image_url blocks was silently string-collapsed before the request left LiteLLM — agent loops that screenshot inside a tool sent pixels that never reached the model. The DeepSeek Chat Completions API itself accepts and reads tool-result images (verified directly against the API, issue #44211). Allow role=tool alongside role=user in the forwardable check; the model capability gate (supports_vision) still lives in the caller.
This commit is contained in:
parent
1f5005ec54
commit
5e6ce0f6db
2 changed files with 78 additions and 6 deletions
|
|
@ -160,13 +160,18 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
|
|||
|
||||
def _is_vision_forwardable_content(self, message: AllMessageValues, content: Sequence[object]) -> bool:
|
||||
"""
|
||||
True only for a user message whose content list holds well-formed
|
||||
text and image_url blocks with at least one image; a block missing
|
||||
its payload falls back to the string collapse instead of crashing
|
||||
or reaching the wire malformed. The model capability gate lives in
|
||||
the caller.
|
||||
True only for a user or tool message whose content list holds
|
||||
well-formed text and image_url blocks with at least one image; a block
|
||||
missing its payload falls back to the string collapse instead of
|
||||
crashing or reaching the wire malformed. The model capability gate
|
||||
lives in the caller.
|
||||
|
||||
``role="tool"`` is forwardable: the DeepSeek Chat Completions API
|
||||
accepts and reads image_url blocks in tool results (verified directly
|
||||
against the API; bug #44211) — agent loops that screenshot inside a
|
||||
tool rely on it.
|
||||
"""
|
||||
if message.get("role") != "user":
|
||||
if message.get("role") not in ("user", "tool"):
|
||||
return False
|
||||
if not all(self._is_forwardable_block(block) for block in content):
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -0,0 +1,67 @@
|
|||
"""
|
||||
Tests for DeepSeek vision-content forwarding in role=tool messages.
|
||||
|
||||
Bug #44211: ``_is_vision_forwardable_content`` rejected every non-user role,
|
||||
so a ``role=tool`` message carrying image_url blocks was silently collapsed
|
||||
to text before the request left LiteLLM — while the DeepSeek API itself
|
||||
accepts and reads tool-result images.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig
|
||||
|
||||
|
||||
TOOL_MESSAGE_WITH_IMAGE = {
|
||||
"role": "tool",
|
||||
"tool_call_id": "abc",
|
||||
"content": [
|
||||
{"type": "text", "text": "screenshot:"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,aGVsbG8="}},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class TestVisionForwardableContent:
|
||||
def test_tool_message_with_image_is_forwardable(self):
|
||||
config = DeepSeekChatConfig()
|
||||
assert config._is_vision_forwardable_content(
|
||||
message=TOOL_MESSAGE_WITH_IMAGE,
|
||||
content=TOOL_MESSAGE_WITH_IMAGE["content"],
|
||||
)
|
||||
|
||||
def test_user_message_with_image_stays_forwardable(self):
|
||||
config = DeepSeekChatConfig()
|
||||
message = {"role": "user", "content": TOOL_MESSAGE_WITH_IMAGE["content"]}
|
||||
assert config._is_vision_forwardable_content(message=message, content=message["content"])
|
||||
|
||||
def test_assistant_message_stays_collapsed(self):
|
||||
config = DeepSeekChatConfig()
|
||||
message = {"role": "assistant", "content": TOOL_MESSAGE_WITH_IMAGE["content"]}
|
||||
assert not config._is_vision_forwardable_content(message=message, content=message["content"])
|
||||
|
||||
def test_image_missing_payload_falls_back(self):
|
||||
config = DeepSeekChatConfig()
|
||||
message = {
|
||||
"role": "tool",
|
||||
"content": [
|
||||
{"type": "text", "text": "screenshot:"},
|
||||
{"type": "image_url", "image_url": {"url": ""}},
|
||||
],
|
||||
}
|
||||
assert not config._is_vision_forwardable_content(message=message, content=message["content"])
|
||||
|
||||
|
||||
class TestForwardOrCollapseContent:
|
||||
def test_tool_message_image_content_is_not_collapsed(self):
|
||||
config = DeepSeekChatConfig()
|
||||
out = config._forward_or_collapse_content(message=TOOL_MESSAGE_WITH_IMAGE, forward_images=True)
|
||||
assert isinstance(out.get("content"), list)
|
||||
blocks = out["content"]
|
||||
assert any(isinstance(block, dict) and block.get("type") == "image_url" for block in blocks)
|
||||
|
||||
def test_tool_message_text_only_still_collapsed(self):
|
||||
config = DeepSeekChatConfig()
|
||||
message = {"role": "tool", "content": [{"type": "text", "text": "plain"}]}
|
||||
out = config._forward_or_collapse_content(message=message, forward_images=True)
|
||||
assert out.get("content") == "plain"
|
||||
Loading…
Add table
Reference in a new issue