From df7b1f0fce595088d740f257e3e34edab4176310 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Wed, 26 Aug 2026 12:19:25 -0700 Subject: [PATCH 1/2] fix: forward image content lists to DeepSeek vision models --- litellm/llms/deepseek/chat/transformation.py | 90 ++++++- ...odel_prices_and_context_window_backup.json | 52 ++++ model_prices_and_context_window.json | 52 ++++ .../chat/test_deepseek_chat_transformation.py | 253 +++++++++++++++++- 4 files changed, 437 insertions(+), 10 deletions(-) diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index 566c960333a..e167819a65c 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -2,16 +2,17 @@ Translates from OpenAI's `/v1/chat/completions` to DeepSeek's `/v1/chat/completions` """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping, Sequence from typing import Any, Final, Literal, cast, overload import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( - handle_messages_with_content_list_to_str_conversion, + convert_content_list_to_str, + extract_search_results_text, ) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues -from litellm.utils import supports_reasoning +from litellm.utils import supports_reasoning, supports_vision from ...openai.chat.gpt_transformation import OpenAIGPTConfig @@ -117,13 +118,88 @@ class DeepSeekChatConfig(OpenAIGPTConfig): self, messages: list[AllMessageValues], model: str, is_async: bool = False ) -> list[AllMessageValues] | Coroutine[Any, Any, list[AllMessageValues]]: """ - DeepSeek does not support content in list format. + DeepSeek vision models accept image_url content blocks in user + messages (https://api-docs.deepseek.com/guides/vision), so those + content lists are forwarded as-is, with any search_results text + appended as a trailing text block. Every other message keeps the + historical string collapse (which also folds search_results text + into string content); a list with no extractable text stays + unchanged, matching what DeepSeek historically received. """ - messages = handle_messages_with_content_list_to_str_conversion(messages) + forward_images: Final = any( + isinstance(message.get("content"), list) for message in messages + ) and supports_vision(model=model, custom_llm_provider="deepseek") + transformed: Final = [ # mutable-ok: provider messages must stay JSON-array lists the base transform mutates + self._forward_or_collapse_content(message=message, forward_images=forward_images) for message in messages + ] + if is_async: - return super()._transform_messages(messages=messages, model=model, is_async=True) + return super()._transform_messages(messages=transformed, model=model, is_async=True) else: - return super()._transform_messages(messages=messages, model=model, is_async=False) + return super()._transform_messages(messages=transformed, model=model, is_async=False) + + def _forward_or_collapse_content(self, message: AllMessageValues, forward_images: bool) -> AllMessageValues: + """ + Returns the vision-forwardable message with any search_results text + appended as a text block; every other message keeps the historical + string collapse, which extracts the text from a content list and + folds search_results text into string content. + """ + content: Final = message.get("content") + if ( + forward_images + and isinstance(content, list) + and self._is_vision_forwardable_content(message=message, content=content) + ): + return self._with_search_results_text_block(message=message, content=content) + collapsed: Final = convert_content_list_to_str(message=message) + if not collapsed or collapsed == content: + return message + collapsed_message: Final = {**message, "content": collapsed} # mutable-ok: wire messages are plain JSON dicts + return cast(AllMessageValues, collapsed_message) # cast-ok: TypedDict spread narrows to dict + + def _is_vision_forwardable_content(self, message: AllMessageValues, content: Sequence[object]) -> bool: + """ + True only for a user message whose content list holds well-formed + text and image_url blocks with at least one image; a block missing + its payload falls back to the string collapse instead of crashing + or reaching the wire malformed. The model capability gate lives in + the caller. + """ + if message.get("role") != "user": + return False + if not all(self._is_forwardable_block(block) for block in content): + return False + return any(isinstance(block, dict) and block.get("type") == "image_url" for block in content) + + @staticmethod + def _is_forwardable_block(block: object) -> bool: + """A dict block typed text or image_url that carries its payload.""" + if not isinstance(block, dict): + return False + block_type: Final = block.get("type") + if block_type == "image_url": + return block.get("image_url") is not None + if block_type == "text": + return isinstance(block.get("text"), str) + return False + + def _with_search_results_text_block(self, message: AllMessageValues, content: Sequence[object]) -> AllMessageValues: + """ + Appends the message's search_results text as a trailing text block, + keeping the context that the string collapse used to fold in, and + drops the non-OpenAI search_results key from the wire message. + """ + message_fields: Final = cast(Mapping[str, object], message) # cast-ok: search_results is not on the TypedDicts + search_text: Final = extract_search_results_text(message_fields.get("search_results")) + if not search_text: + return message + forwarded_content: Final = [*content, {"type": "text", "text": search_text}] # mutable-ok: JSON-array content + forwarded: Final = { # mutable-ok: wire messages are plain JSON dicts + **{key: value for key, value in message_fields.items() if key != "search_results"}, + "content": forwarded_content, + } + return cast(AllMessageValues, forwarded) # cast-ok: TypedDict spread narrows to dict def _thinking_mode_active(self, model: str, optional_params: dict) -> bool: """ diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c05aefee15e..22572f7f20e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -50413,6 +50413,32 @@ "supports_tool_choice": true, "supports_vision": false }, + "deepseek-v4-flash-vision-exp": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, + "litellm_provider": "deepseek", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://api-docs.deepseek.com/quick_start/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 4.4e-08, @@ -50465,6 +50491,32 @@ "supports_tool_choice": true, "supports_vision": false }, + "deepseek/deepseek-v4-flash-vision-exp": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, + "litellm_provider": "deepseek", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://api-docs.deepseek.com/quick_start/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "deepseek/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 4.4e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c05aefee15e..22572f7f20e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -50413,6 +50413,32 @@ "supports_tool_choice": true, "supports_vision": false }, + "deepseek-v4-flash-vision-exp": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, + "litellm_provider": "deepseek", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://api-docs.deepseek.com/quick_start/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 4.4e-08, @@ -50465,6 +50491,32 @@ "supports_tool_choice": true, "supports_vision": false }, + "deepseek/deepseek-v4-flash-vision-exp": { + "cache_creation_input_token_cost": 0.0, + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "input_cost_per_token_cache_hit": 1.4e-08, + "litellm_provider": "deepseek", + "max_input_tokens": 1000000, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "source": "https://api-docs.deepseek.com/quick_start/pricing", + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": true + }, "deepseek/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 4.4e-08, diff --git a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py index fa6f23dc7ff..ba6ee4d6663 100644 --- a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -1,3 +1,4 @@ +import litellm from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig @@ -108,6 +109,254 @@ def test_thinking_mode_active_bool_thinking_returns_false_without_crashing(): assert config._thinking_mode_active(model="deepseek-reasoner", optional_params={"thinking": True}) is False +class TestDeepSeekVisionMultimodalContent: + """Image content lists are forwarded only for user messages on vision models.""" + + VISION_MODEL = "deepseek/deepseek-v4-flash-vision-exp" + NON_VISION_MODEL = "deepseek/deepseek-chat" + + def setup_method(self): + self.config = DeepSeekChatConfig() + prior_entry = litellm.model_cost.get(self.VISION_MODEL) + self._prior_registry_entry = dict(prior_entry) if prior_entry is not None else None + litellm.register_model( + { + "deepseek/deepseek-v4-flash-vision-exp": { + "litellm_provider": "deepseek", + "mode": "chat", + "input_cost_per_token": 4.4e-07, + "output_cost_per_token": 1.32e-06, + "supports_vision": True, + } + } + ) + + def teardown_method(self): + if self._prior_registry_entry is None: + litellm.model_cost.pop(self.VISION_MODEL, None) + else: + litellm.model_cost[self.VISION_MODEL] = self._prior_registry_entry + + @staticmethod + def _image_message(role="user"): + return { + "role": role, + "content": [ + {"type": "text", "text": "what is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.jpg", "detail": "auto"}, + }, + ], + } + + def test_user_image_list_forwarded_on_vision_model(self): + result = self.config._transform_messages([self._image_message()], model=self.VISION_MODEL) + + assert isinstance(result[0]["content"], list) + assert result[0]["content"][0]["type"] == "text" + assert result[0]["content"][1]["type"] == "image_url" + assert result[0]["content"][1]["image_url"]["url"] == "https://example.com/image.jpg" + + def test_image_list_collapsed_on_non_vision_model(self): + result = self.config._transform_messages([self._image_message()], model=self.NON_VISION_MODEL) + + assert result[0]["content"] == "what is in this image?" + + def test_image_list_collapsed_on_non_user_roles_even_on_vision_model(self): + for role in ("assistant", "system"): + result = self.config._transform_messages([self._image_message(role=role)], model=self.VISION_MODEL) + + assert result[0]["content"] == "what is in this image?" + + def test_audio_block_collapsed_even_on_vision_model(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "transcribe this"}, + {"type": "input_audio", "input_audio": {"data": "UklGRg==", "format": "wav"}}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "transcribe this" + + def test_typeless_image_block_collapses(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this"}, + {"image_url": {"url": "https://example.com/image.jpg"}}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "what is this" + + def test_text_only_content_list_collapses(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello "}, + {"type": "text", "text": "world"}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert isinstance(result[0]["content"], str) + assert result[0]["content"] == "Hello world" + + def test_search_results_text_appended_on_forwarded_message(self): + message = self._image_message() + message["search_results"] = [{"source": "kb", "content": [{"text": "article body"}]}] + + result = self.config._transform_messages([message], model=self.VISION_MODEL) + + content = result[0]["content"] + assert isinstance(content, list) + assert content[-1] == {"type": "text", "text": "kbarticle body"} + assert any(block.get("type") == "image_url" for block in content) + assert "search_results" not in result[0] + + def test_search_results_text_kept_on_collapse(self): + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "context: "}], + "search_results": [{"source": "kb", "content": [{"text": "article body"}]}], + } + ] + + result = self.config._transform_messages(messages, model=self.NON_VISION_MODEL) + + assert result[0]["content"] == "context: kbarticle body" + + def test_responses_shape_blocks_collapse_even_on_vision_model(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "what is this?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "what is this?" + + def test_image_block_missing_payload_collapses(self): + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "hi"}, {"type": "image_url"}], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "hi" + + def test_text_block_missing_text_field_collapses(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "hi"}, + {"type": "text"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "hi" + + def test_string_content_search_results_folded_into_string(self): + messages = [ + { + "role": "tool", + "tool_call_id": "call_1", + "content": "summarize the docs", + "search_results": [{"source": "kb", "content": [{"text": "article body"}]}], + } + ] + + result = self.config._transform_messages(messages, model=self.NON_VISION_MODEL) + + assert result[0]["content"] == "summarize the docskbarticle body" + + def test_plain_string_content_message_unchanged(self): + messages = [{"role": "user", "content": "hello"}] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0] is messages[0] + + def test_empty_content_list_untouched(self): + messages = [{"role": "user", "content": []}] + + result = self.config._transform_messages(messages, model=self.NON_VISION_MODEL) + + assert result[0]["content"] == [] + + def test_later_messages_still_collapsed_after_forwarded_one(self): + messages = [ + self._image_message(), + { + "role": "user", + "content": [ + {"type": "text", "text": "and "}, + {"type": "text", "text": "then?"}, + ], + }, + self._image_message(), + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert isinstance(result[0]["content"], list) + assert result[1]["content"] == "and then?" + assert isinstance(result[2]["content"], list) + + def test_transform_request_preserves_image_url_block(self): + body = self.config.transform_request( + model=self.VISION_MODEL, + messages=[self._image_message()], + optional_params={}, + litellm_params={}, + headers={}, + ) + + content = body["messages"][0]["content"] + assert isinstance(content, list) + assert any(block.get("type") == "image_url" for block in content) + + async def test_async_transform_request_preserves_image_url_block(self): + body = await self.config.async_transform_request( + model=self.VISION_MODEL, + messages=[self._image_message()], + optional_params={}, + litellm_params={}, + headers={}, + ) + + content = body["messages"][0]["content"] + assert isinstance(content, list) + assert any(block.get("type") == "image_url" for block in content) + + class TestDeepSeekThinkingParams: """Test thinking and reasoning_effort parameter handling for DeepSeek.""" @@ -282,8 +531,6 @@ class TestDeepSeekThinkingParams: result = self.config._drop_unsupported_tools(optional_params) - assert result["tools"] == [ - {"type": "function", "function": {"name": "get_weather"}} - ] + assert result["tools"] == [{"type": "function", "function": {"name": "get_weather"}}] assert "tool_choice" not in result assert result["parallel_tool_calls"] is True From c3c9903ba5222f046a4e544d4998e18273b90219 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Wed, 26 Aug 2026 13:19:06 -0700 Subject: [PATCH 2/2] fix: collapse image_url blocks whose payload lacks a url instead of forwarding them --- litellm/llms/deepseek/chat/transformation.py | 12 +++++++- .../chat/test_deepseek_chat_transformation.py | 30 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index e167819a65c..ea19a7c7ddf 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -179,11 +179,21 @@ class DeepSeekChatConfig(OpenAIGPTConfig): return False block_type: Final = block.get("type") if block_type == "image_url": - return block.get("image_url") is not None + return DeepSeekChatConfig._is_image_url_payload(block.get("image_url")) if block_type == "text": return isinstance(block.get("text"), str) return False + @staticmethod + def _is_image_url_payload(payload: object) -> bool: + """A url string or an object carrying one, per the OpenAI image_url shape.""" + if isinstance(payload, str): + return bool(payload) + if not isinstance(payload, Mapping): + return False + url: Final = payload.get("url") + return isinstance(url, str) and bool(url) + def _with_search_results_text_block(self, message: AllMessageValues, content: Sequence[object]) -> AllMessageValues: """ Appends the message's search_results text as a trailing text block, diff --git a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py index ba6ee4d6663..3f93264d0d0 100644 --- a/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/test_litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -267,6 +267,36 @@ class TestDeepSeekVisionMultimodalContent: assert result[0]["content"] == "hi" + def test_image_block_empty_payload_object_collapses(self): + for payload in ({}, {"url": ""}, {"detail": "auto"}, None, 42): + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "hi"}, {"type": "image_url", "image_url": payload}], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + assert result[0]["content"] == "hi" + + def test_image_block_string_payload_forwarded(self): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this?"}, + {"type": "image_url", "image_url": "https://example.com/image.jpg"}, + ], + } + ] + + result = self.config._transform_messages(messages, model=self.VISION_MODEL) + + content = result[0]["content"] + assert isinstance(content, list) + assert content[1]["image_url"] == {"url": "https://example.com/image.jpg"} + def test_text_block_missing_text_field_collapses(self): messages = [ {