Merge pull request #38397 from BerriAI/litellm_deepseek_vision_forwarding

fix: forward image content lists to DeepSeek vision models

litellm/llms/deepseek/chat/transformation.py

@@ -2,16 +2,17 @@
 Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completions`
 """
 
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping, Sequence
 from typing import Any, Final, Literal, cast, overload
 
 import litellm
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
-    handle_messages_with_content_list_to_str_conversion,
+    convert_content_list_to_str,
+    extract_search_results_text,
 )
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
-from litellm.utils import supports_reasoning
+from litellm.utils import supports_reasoning, supports_vision
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
@@ -117,13 +118,98 @@ class DeepSeekChatConfig(OpenAIGPTConfig):
         self, messages: list[AllMessageValues], model: str, is_async: bool = False
     ) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]:
         """
-        DeepSeek does not support content in list format.
+        DeepSeek vision models accept image_url content blocks in user
+        messages (https://api-docs.deepseek.com/guides/vision), so those
+        content lists are forwarded as-is, with any search_results text
+        appended as a trailing text block. Every other message keeps the
+        historical string collapse (which also folds search_results text
+        into string content); a list with no extractable text stays
+        unchanged, matching what DeepSeek historically received.
         """
-        messages = handle_messages_with_content_list_to_str_conversion(messages)
+        forward_images: Final = any(
+            isinstance(message.get("content"), list) for message in messages
+        ) and supports_vision(model=model, custom_llm_provider="deepseek")
+        transformed: Final = [  # mutable-ok: provider messages must stay JSON-array lists the base transform mutates
+            self._forward_or_collapse_content(message=message, forward_images=forward_images) for message in messages
+        ]
+
         if is_async:
-            return super()._transform_messages(messages=messages, model=model, is_async=True)
+            return super()._transform_messages(messages=transformed, model=model, is_async=True)
         else:
-            return super()._transform_messages(messages=messages, model=model, is_async=False)
+            return super()._transform_messages(messages=transformed, model=model, is_async=False)
+
+    def _forward_or_collapse_content(self, message: AllMessageValues, forward_images: bool) -> AllMessageValues:
+        """
+        Returns the vision-forwardable message with any search_results text
+        appended as a text block; every other message keeps the historical
+        string collapse, which extracts the text from a content list and
+        folds search_results text into string content.
+        """
+        content: Final = message.get("content")
+        if (
+            forward_images
+            and isinstance(content, list)
+            and self._is_vision_forwardable_content(message=message, content=content)
+        ):
+            return self._with_search_results_text_block(message=message, content=content)
+        collapsed: Final = convert_content_list_to_str(message=message)
+        if not collapsed or collapsed == content:
+            return message
+        collapsed_message: Final = {**message, "content": collapsed}  # mutable-ok: wire messages are plain JSON dicts
+        return cast(AllMessageValues, collapsed_message)  # cast-ok: TypedDict spread narrows to dict
+
+    def _is_vision_forwardable_content(self, message: AllMessageValues, content: Sequence[object]) -> bool:
+        """
+        True only for a user message whose content list holds well-formed
+        text and image_url blocks with at least one image; a block missing
+        its payload falls back to the string collapse instead of crashing
+        or reaching the wire malformed. The model capability gate lives in
+        the caller.
+        """
+        if message.get("role") != "user":
+            return False
+        if not all(self._is_forwardable_block(block) for block in content):
+            return False
+        return any(isinstance(block, dict) and block.get("type") == "image_url" for block in content)
+
+    @staticmethod
+    def _is_forwardable_block(block: object) -> bool:
+        """A dict block typed text or image_url that carries its payload."""
+        if not isinstance(block, dict):
+            return False
+        block_type: Final = block.get("type")
+        if block_type == "image_url":
+            return DeepSeekChatConfig._is_image_url_payload(block.get("image_url"))
+        if block_type == "text":
+            return isinstance(block.get("text"), str)
+        return False
+
+    @staticmethod
+    def _is_image_url_payload(payload: object) -> bool:
+        """A url string or an object carrying one, per the OpenAI image_url shape."""
+        if isinstance(payload, str):
+            return bool(payload)
+        if not isinstance(payload, Mapping):
+            return False
+        url: Final = payload.get("url")
+        return isinstance(url, str) and bool(url)
+
+    def _with_search_results_text_block(self, message: AllMessageValues, content: Sequence[object]) -> AllMessageValues:
+        """
+        Appends the message's search_results text as a trailing text block,
+        keeping the context that the string collapse used to fold in, and
+        drops the non-OpenAI search_results key from the wire message.
+        """
+        message_fields: Final = cast(Mapping[str, object], message)  # cast-ok: search_results is not on the TypedDicts
+        search_text: Final = extract_search_results_text(message_fields.get("search_results"))
+        if not search_text:
+            return message
+        forwarded_content: Final = [*content, {"type": "text", "text": search_text}]  # mutable-ok: JSON-array content
+        forwarded: Final = {  # mutable-ok: wire messages are plain JSON dicts
+            **{key: value for key, value in message_fields.items() if key != "search_results"},
+            "content": forwarded_content,
+        }
+        return cast(AllMessageValues, forwarded)  # cast-ok: TypedDict spread narrows to dict
 
     def _thinking_mode_active(self, model: str, optional_params: dict) -> bool:
         """

litellm/model_prices_and_context_window_backup.json

@@ -50525,6 +50525,32 @@
         "supports_tool_choice": true,
         "supports_vision": false
     },
+    "deepseek-v4-flash-vision-exp": {
+        "cache_creation_input_token_cost": 0.0,
+        "cache_read_input_token_cost": 1.4e-08,
+        "input_cost_per_token": 4.4e-07,
+        "input_cost_per_token_cache_hit": 1.4e-08,
+        "litellm_provider": "deepseek",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 393216,
+        "max_tokens": 393216,
+        "mode": "chat",
+        "output_cost_per_token": 1.32e-06,
+        "source": "https://api-docs.deepseek.com/quick_start/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "deepseek-v4-pro": {
         "cache_creation_input_token_cost": 0.0,
         "cache_read_input_token_cost": 4.4e-08,
@@ -50577,6 +50603,32 @@
         "supports_tool_choice": true,
         "supports_vision": false
     },
+    "deepseek/deepseek-v4-flash-vision-exp": {
+        "cache_creation_input_token_cost": 0.0,
+        "cache_read_input_token_cost": 1.4e-08,
+        "input_cost_per_token": 4.4e-07,
+        "input_cost_per_token_cache_hit": 1.4e-08,
+        "litellm_provider": "deepseek",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 393216,
+        "max_tokens": 393216,
+        "mode": "chat",
+        "output_cost_per_token": 1.32e-06,
+        "source": "https://api-docs.deepseek.com/quick_start/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "deepseek/deepseek-v4-pro": {
         "cache_creation_input_token_cost": 0.0,
         "cache_read_input_token_cost": 4.4e-08,

model_prices_and_context_window.json

@@ -50525,6 +50525,32 @@
         "supports_tool_choice": true,
         "supports_vision": false
     },
+    "deepseek-v4-flash-vision-exp": {
+        "cache_creation_input_token_cost": 0.0,
+        "cache_read_input_token_cost": 1.4e-08,
+        "input_cost_per_token": 4.4e-07,
+        "input_cost_per_token_cache_hit": 1.4e-08,
+        "litellm_provider": "deepseek",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 393216,
+        "max_tokens": 393216,
+        "mode": "chat",
+        "output_cost_per_token": 1.32e-06,
+        "source": "https://api-docs.deepseek.com/quick_start/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "deepseek-v4-pro": {
         "cache_creation_input_token_cost": 0.0,
         "cache_read_input_token_cost": 4.4e-08,
@@ -50577,6 +50603,32 @@
         "supports_tool_choice": true,
         "supports_vision": false
     },
+    "deepseek/deepseek-v4-flash-vision-exp": {
+        "cache_creation_input_token_cost": 0.0,
+        "cache_read_input_token_cost": 1.4e-08,
+        "input_cost_per_token": 4.4e-07,
+        "input_cost_per_token_cache_hit": 1.4e-08,
+        "litellm_provider": "deepseek",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 393216,
+        "max_tokens": 393216,
+        "mode": "chat",
+        "output_cost_per_token": 1.32e-06,
+        "source": "https://api-docs.deepseek.com/quick_start/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "deepseek/deepseek-v4-pro": {
         "cache_creation_input_token_cost": 0.0,
         "cache_read_input_token_cost": 4.4e-08,

tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py

@@ -1,3 +1,4 @@
+import litellm
 from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig
 
 
@@ -108,6 +109,284 @@ def test_thinking_mode_active_bool_thinking_returns_false_without_crashing():
     assert config._thinking_mode_active(model="deepseek-reasoner", optional_params={"thinking": True}) is False
 
 
+class TestDeepSeekVisionMultimodalContent:
+    """Image content lists are forwarded only for user messages on vision models."""
+
+    VISION_MODEL = "deepseek/deepseek-v4-flash-vision-exp"
+    NON_VISION_MODEL = "deepseek/deepseek-chat"
+
+    def setup_method(self):
+        self.config = DeepSeekChatConfig()
+        prior_entry = litellm.model_cost.get(self.VISION_MODEL)
+        self._prior_registry_entry = dict(prior_entry) if prior_entry is not None else None
+        litellm.register_model(
+            {
+                "deepseek/deepseek-v4-flash-vision-exp": {
+                    "litellm_provider": "deepseek",
+                    "mode": "chat",
+                    "input_cost_per_token": 4.4e-07,
+                    "output_cost_per_token": 1.32e-06,
+                    "supports_vision": True,
+                }
+            }
+        )
+
+    def teardown_method(self):
+        if self._prior_registry_entry is None:
+            litellm.model_cost.pop(self.VISION_MODEL, None)
+        else:
+            litellm.model_cost[self.VISION_MODEL] = self._prior_registry_entry
+
+    @staticmethod
+    def _image_message(role="user"):
+        return {
+            "role": role,
+            "content": [
+                {"type": "text", "text": "what is in this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://example.com/image.jpg", "detail": "auto"},
+                },
+            ],
+        }
+
+    def test_user_image_list_forwarded_on_vision_model(self):
+        result = self.config._transform_messages([self._image_message()], model=self.VISION_MODEL)
+
+        assert isinstance(result[0]["content"], list)
+        assert result[0]["content"][0]["type"] == "text"
+        assert result[0]["content"][1]["type"] == "image_url"
+        assert result[0]["content"][1]["image_url"]["url"] == "https://example.com/image.jpg"
+
+    def test_image_list_collapsed_on_non_vision_model(self):
+        result = self.config._transform_messages([self._image_message()], model=self.NON_VISION_MODEL)
+
+        assert result[0]["content"] == "what is in this image?"
+
+    def test_image_list_collapsed_on_non_user_roles_even_on_vision_model(self):
+        for role in ("assistant", "system"):
+            result = self.config._transform_messages([self._image_message(role=role)], model=self.VISION_MODEL)
+
+            assert result[0]["content"] == "what is in this image?"
+
+    def test_audio_block_collapsed_even_on_vision_model(self):
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "transcribe this"},
+                    {"type": "input_audio", "input_audio": {"data": "UklGRg==", "format": "wav"}},
+                ],
+            }
+        ]
+
+        result = self.config._transform_messages(messages, model=self.VISION_MODEL)
+
+        assert result[0]["content"] == "transcribe this"
+
+    def test_typeless_image_block_collapses(self):
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "what is this"},
+                    {"image_url": {"url": "https://example.com/image.jpg"}},
+                ],
+            }
+        ]
+
+        result = self.config._transform_messages(messages, model=self.VISION_MODEL)
+
+        assert result[0]["content"] == "what is this"
+
+    def test_text_only_content_list_collapses(self):
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "Hello "},
+                    {"type": "text", "text": "world"},
+                ],
+            }
+        ]
+
+        result = self.config._transform_messages(messages, model=self.VISION_MODEL)
+
+        assert isinstance(result[0]["content"], str)
+        assert result[0]["content"] == "Hello world"
+
+    def test_search_results_text_appended_on_forwarded_message(self):
+        message = self._image_message()
+        message["search_results"] = [{"source": "kb", "content": [{"text": "article body"}]}]
+
+        result = self.config._transform_messages([message], model=self.VISION_MODEL)
+
+        content = result[0]["content"]
+        assert isinstance(content, list)
+        assert content[-1] == {"type": "text", "text": "kbarticle body"}
+        assert any(block.get("type") == "image_url" for block in content)
+        assert "search_results" not in result[0]
+
+    def test_search_results_text_kept_on_collapse(self):
+        messages = [
+            {
+                "role": "user",
+                "content": [{"type": "text", "text": "context: "}],
+                "search_results": [{"source": "kb", "content": [{"text": "article body"}]}],
+            }
+        ]
+
+        result = self.config._transform_messages(messages, model=self.NON_VISION_MODEL)
+
+        assert result[0]["content"] == "context: kbarticle body"
+
+    def test_responses_shape_blocks_collapse_even_on_vision_model(self):
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "input_text", "text": "what is this?"},
+                    {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}},
+                ],
+            }
+        ]
+
+        result = self.config._transform_messages(messages, model=self.VISION_MODEL)
+
+        assert result[0]["content"] == "what is this?"
+
+    def test_image_block_missing_payload_collapses(self):
+        messages = [
+            {
+                "role": "user",
+                "content": [{"type": "text", "text": "hi"}, {"type": "image_url"}],
+            }
+        ]
+
+        result = self.config._transform_messages(messages, model=self.VISION_MODEL)
+
+        assert result[0]["content"] == "hi"
+
+    def test_image_block_empty_payload_object_collapses(self):
+        for payload in ({}, {"url": ""}, {"detail": "auto"}, None, 42):
+            messages = [
+                {
+                    "role": "user",
+                    "content": [{"type": "text", "text": "hi"}, {"type": "image_url", "image_url": payload}],
+                }
+            ]
+
+            result = self.config._transform_messages(messages, model=self.VISION_MODEL)
+
+            assert result[0]["content"] == "hi"
+
+    def test_image_block_string_payload_forwarded(self):
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "what is this?"},
+                    {"type": "image_url", "image_url": "https://example.com/image.jpg"},
+                ],
+            }
+        ]
+
+        result = self.config._transform_messages(messages, model=self.VISION_MODEL)
+
+        content = result[0]["content"]
+        assert isinstance(content, list)
+        assert content[1]["image_url"] == {"url": "https://example.com/image.jpg"}
+
+    def test_text_block_missing_text_field_collapses(self):
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "hi"},
+                    {"type": "text"},
+                    {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}},
+                ],
+            }
+        ]
+
+        result = self.config._transform_messages(messages, model=self.VISION_MODEL)
+
+        assert result[0]["content"] == "hi"
+
+    def test_string_content_search_results_folded_into_string(self):
+        messages = [
+            {
+                "role": "tool",
+                "tool_call_id": "call_1",
+                "content": "summarize the docs",
+                "search_results": [{"source": "kb", "content": [{"text": "article body"}]}],
+            }
+        ]
+
+        result = self.config._transform_messages(messages, model=self.NON_VISION_MODEL)
+
+        assert result[0]["content"] == "summarize the docskbarticle body"
+
+    def test_plain_string_content_message_unchanged(self):
+        messages = [{"role": "user", "content": "hello"}]
+
+        result = self.config._transform_messages(messages, model=self.VISION_MODEL)
+
+        assert result[0] is messages[0]
+
+    def test_empty_content_list_untouched(self):
+        messages = [{"role": "user", "content": []}]
+
+        result = self.config._transform_messages(messages, model=self.NON_VISION_MODEL)
+
+        assert result[0]["content"] == []
+
+    def test_later_messages_still_collapsed_after_forwarded_one(self):
+        messages = [
+            self._image_message(),
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "and "},
+                    {"type": "text", "text": "then?"},
+                ],
+            },
+            self._image_message(),
+        ]
+
+        result = self.config._transform_messages(messages, model=self.VISION_MODEL)
+
+        assert isinstance(result[0]["content"], list)
+        assert result[1]["content"] == "and then?"
+        assert isinstance(result[2]["content"], list)
+
+    def test_transform_request_preserves_image_url_block(self):
+        body = self.config.transform_request(
+            model=self.VISION_MODEL,
+            messages=[self._image_message()],
+            optional_params={},
+            litellm_params={},
+            headers={},
+        )
+
+        content = body["messages"][0]["content"]
+        assert isinstance(content, list)
+        assert any(block.get("type") == "image_url" for block in content)
+
+    async def test_async_transform_request_preserves_image_url_block(self):
+        body = await self.config.async_transform_request(
+            model=self.VISION_MODEL,
+            messages=[self._image_message()],
+            optional_params={},
+            litellm_params={},
+            headers={},
+        )
+
+        content = body["messages"][0]["content"]
+        assert isinstance(content, list)
+        assert any(block.get("type") == "image_url" for block in content)
+
+
 class TestDeepSeekThinkingParams:
     """Test thinking and reasoning_effort parameter handling for DeepSeek."""
 
@@ -282,8 +561,6 @@ class TestDeepSeekThinkingParams:
 
         result = self.config._drop_unsupported_tools(optional_params)
 
-        assert result["tools"] == [
-            {"type": "function", "function": {"name": "get_weather"}}
-        ]
+        assert result["tools"] == [{"type": "function", "function": {"name": "get_weather"}}]
         assert "tool_choice" not in result
         assert result["parallel_tool_calls"] is True