fix(meta): normalize generic image data-URL mime types for muse-spark vision requests

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -2,6 +2,7 @@
 Common utility functions used for translating messages across providers
 """
 
+import base64
 import io
 import json
 import mimetypes
@@ -31,6 +32,7 @@ from litellm.types.llms.openai import (
     ChatCompletionResponseMessage,
     ChatCompletionToolParam,
     ChatCompletionUserMessage,
+    OpenAIMessageContentListBlock,
 )
 from litellm.types.utils import (
     Choices,
@@ -1204,6 +1206,86 @@ def infer_content_type_from_url_and_content(
     raise ValueError(f"Unable to determine content type from URL: {url}. Response content-type: {current_content_type}")
 
 
+_GENERIC_DATA_URL_MIME_TYPES = frozenset({"", "binary/octet-stream", "application/octet-stream"})
+
+_SNIFFED_IMAGE_TYPE_TO_MIME_TYPE: Mapping[str, str] = {
+    "png": "image/png",
+    "jpeg": "image/jpeg",
+    "gif": "image/gif",
+    "webp": "image/webp",
+    "heic": "image/heic",
+}
+
+
+def _sniff_base64_image_mime_type(base64_payload: str) -> str | None:
+    from litellm.litellm_core_utils.token_counter import get_image_type
+
+    prefix = base64_payload[:64]
+    aligned_prefix = prefix[: len(prefix) - len(prefix) % 4]
+    if not aligned_prefix:
+        return None
+    try:
+        decoded_prefix = base64.b64decode(aligned_prefix)
+    except ValueError:
+        return None
+    sniffed_image_type = get_image_type(decoded_prefix)
+    if sniffed_image_type is None:
+        return None
+    return _SNIFFED_IMAGE_TYPE_TO_MIME_TYPE.get(sniffed_image_type)
+
+
+def normalize_image_data_url_mime_type(url: str) -> str:
+    if not url.startswith("data:") or ";base64," not in url:
+        return url
+    header, payload = url.split(";base64,", 1)
+    declared_mime_type = header[len("data:") :]
+    if declared_mime_type.lower() not in _GENERIC_DATA_URL_MIME_TYPES:
+        return url
+    sniffed_mime_type = _sniff_base64_image_mime_type(payload)
+    if sniffed_mime_type is None:
+        return url
+    return f"data:{sniffed_mime_type};base64,{payload}"
+
+
+def _normalize_image_mime_type_in_content_item(
+    content_item: OpenAIMessageContentListBlock,
+) -> OpenAIMessageContentListBlock:
+    if content_item["type"] != "image_url":
+        return content_item
+    image_url = content_item.get("image_url")
+    if isinstance(image_url, str):
+        normalized_item = content_item.copy()
+        normalized_item["image_url"] = normalize_image_data_url_mime_type(image_url)
+        return normalized_item
+    if not isinstance(image_url, dict) or "url" not in image_url:
+        return content_item
+    url = image_url["url"]
+    if not isinstance(url, str):
+        return content_item
+    normalized_image_url = image_url.copy()
+    normalized_image_url["url"] = normalize_image_data_url_mime_type(url)
+    normalized_item = content_item.copy()
+    normalized_item["image_url"] = normalized_image_url
+    return normalized_item
+
+
+def _normalize_image_mime_types_in_message(message: AllMessageValues) -> AllMessageValues:
+    if message["role"] != "user":
+        return message
+    content = message.get("content")
+    if not isinstance(content, list):
+        return message
+    normalized_message = message.copy()
+    normalized_message["content"] = [_normalize_image_mime_type_in_content_item(item) for item in content]
+    return normalized_message
+
+
+def normalize_image_data_url_mime_types_in_messages(
+    messages: list[AllMessageValues],
+) -> list[AllMessageValues]:
+    return [_normalize_image_mime_types_in_message(message) for message in messages]
+
+
 def get_tool_call_names(tools: List[ChatCompletionToolParam]) -> List[str]:
     """
     Get tool call names from tools

litellm/llms/openai_like/dynamic_config.py

@@ -7,6 +7,7 @@ from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overlo
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     handle_messages_with_content_list_to_str_conversion,
+    normalize_image_data_url_mime_types_in_messages,
 )
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
 from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
@@ -45,6 +46,9 @@ def create_config_class(provider: SimpleProviderConfig):
             if provider.special_handling.get("convert_content_list_to_string"):
                 messages = handle_messages_with_content_list_to_str_conversion(messages)
 
+            if provider.special_handling.get("normalize_image_mime_type"):
+                messages = normalize_image_data_url_mime_types_in_messages(messages)
+
             if is_async:
                 return super()._transform_messages(messages=messages, model=model, is_async=True)
             else:

litellm/llms/openai_like/providers.json

@@ -173,7 +173,10 @@
     "api_key_env": "META_API_KEY",
     "api_base_env": "META_API_BASE",
     "base_class": "openai_gpt",
-    "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
+    "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"],
+    "special_handling": {
+      "normalize_image_mime_type": true
+    }
   },
   "pinstripes": {
     "base_url": "https://pinstripes.io/v1",

tests/test_litellm/llms/openai_like/test_meta_provider.py

@@ -2,6 +2,9 @@
 Tests for the Meta Model API (Muse Spark) provider configuration and integration.
 """
 
+import asyncio
+import base64
+
 import litellm
 
 
@@ -192,6 +195,101 @@ class TestMetaAnthropicMessages:
         assert headers["anthropic-version"] == "2023-06-01"
 
 
+PNG_BASE64 = base64.b64encode(b"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a" + b"\x00" * 32).decode()
+JPEG_BASE64 = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 32).decode()
+
+
+def _transform_image_url_through_provider(url: str, provider: litellm.LlmProviders) -> str:
+    cfg = litellm.ProviderConfigManager.get_provider_chat_config(model="muse-spark-1.1", provider=provider)
+    assert cfg is not None
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is in this image?"},
+                {"type": "image_url", "image_url": {"url": url}},
+            ],
+        }
+    ]
+    out = cfg.transform_request(
+        model="muse-spark-1.1",
+        messages=messages,
+        optional_params={},
+        litellm_params={},
+        headers={},
+    )
+    return out["messages"][0]["content"][1]["image_url"]["url"]
+
+
+class TestMetaImageMimeNormalization:
+    def test_meta_config_has_normalize_flag(self):
+        from litellm.llms.openai_like.json_loader import JSONProviderRegistry
+
+        meta = JSONProviderRegistry.get("meta")
+        assert meta is not None
+        assert meta.special_handling.get("normalize_image_mime_type") is True
+
+    def test_octet_stream_png_rewritten(self):
+        url = _transform_image_url_through_provider(
+            f"data:binary/octet-stream;base64,{PNG_BASE64}", litellm.LlmProviders.META
+        )
+        assert url == f"data:image/png;base64,{PNG_BASE64}"
+
+    def test_application_octet_stream_jpeg_rewritten(self):
+        url = _transform_image_url_through_provider(
+            f"data:application/octet-stream;base64,{JPEG_BASE64}", litellm.LlmProviders.META
+        )
+        assert url == f"data:image/jpeg;base64,{JPEG_BASE64}"
+
+    def test_valid_mime_untouched(self):
+        original = f"data:image/jpeg;base64,{JPEG_BASE64}"
+        assert _transform_image_url_through_provider(original, litellm.LlmProviders.META) == original
+
+    def test_mismatched_but_valid_mime_untouched(self):
+        original = f"data:image/jpeg;base64,{PNG_BASE64}"
+        assert _transform_image_url_through_provider(original, litellm.LlmProviders.META) == original
+
+    def test_undecodable_payload_untouched(self):
+        original = "data:binary/octet-stream;base64,!!!not-base64!!!"
+        assert _transform_image_url_through_provider(original, litellm.LlmProviders.META) == original
+
+    def test_http_url_untouched(self):
+        original = "https://example.com/image.png"
+        assert _transform_image_url_through_provider(original, litellm.LlmProviders.META) == original
+
+    def test_provider_without_flag_untouched(self):
+        original = f"data:binary/octet-stream;base64,{PNG_BASE64}"
+        assert _transform_image_url_through_provider(original, litellm.LlmProviders.TENSORMESH) == original
+
+    def test_async_transform_rewrites_octet_stream_png(self):
+        cfg = litellm.ProviderConfigManager.get_provider_chat_config(
+            model="muse-spark-1.1", provider=litellm.LlmProviders.META
+        )
+        assert cfg is not None
+        messages = [
+            {
+                "role": "user",
+                "content": [{"type": "image_url", "image_url": {"url": f"data:binary/octet-stream;base64,{PNG_BASE64}"}}],
+            }
+        ]
+        out = asyncio.run(cfg._transform_messages(messages=messages, model="muse-spark-1.1", is_async=True))
+        assert out[0]["content"][0]["image_url"]["url"] == f"data:image/png;base64,{PNG_BASE64}"
+
+    def test_string_content_passthrough(self):
+        cfg = litellm.ProviderConfigManager.get_provider_chat_config(
+            model="muse-spark-1.1", provider=litellm.LlmProviders.META
+        )
+        assert cfg is not None
+        out = cfg.transform_request(
+            model="muse-spark-1.1",
+            messages=[{"role": "user", "content": "hello"}],
+            optional_params={},
+            litellm_params={},
+            headers={},
+        )
+        assert out["messages"][0]["content"] == "hello"
+
+
 class TestMuseSparkModelInfo:
     def test_muse_spark_pricing_and_capabilities(self):
         info = litellm.get_model_info("meta/muse-spark-1.1")