diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 538d5f650ef..aabd1d86d81 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2,6 +2,7 @@ Common utility functions used for translating messages across providers """ +import base64 import io import json import mimetypes @@ -31,6 +32,7 @@ from litellm.types.llms.openai import ( ChatCompletionResponseMessage, ChatCompletionToolParam, ChatCompletionUserMessage, + OpenAIMessageContentListBlock, ) from litellm.types.utils import ( Choices, @@ -1204,6 +1206,86 @@ def infer_content_type_from_url_and_content( raise ValueError(f"Unable to determine content type from URL: {url}. Response content-type: {current_content_type}") +_GENERIC_DATA_URL_MIME_TYPES = frozenset({"", "binary/octet-stream", "application/octet-stream"}) + +_SNIFFED_IMAGE_TYPE_TO_MIME_TYPE: Mapping[str, str] = { + "png": "image/png", + "jpeg": "image/jpeg", + "gif": "image/gif", + "webp": "image/webp", + "heic": "image/heic", +} + + +def _sniff_base64_image_mime_type(base64_payload: str) -> str | None: + from litellm.litellm_core_utils.token_counter import get_image_type + + prefix = base64_payload[:64] + aligned_prefix = prefix[: len(prefix) - len(prefix) % 4] + if not aligned_prefix: + return None + try: + decoded_prefix = base64.b64decode(aligned_prefix) + except ValueError: + return None + sniffed_image_type = get_image_type(decoded_prefix) + if sniffed_image_type is None: + return None + return _SNIFFED_IMAGE_TYPE_TO_MIME_TYPE.get(sniffed_image_type) + + +def normalize_image_data_url_mime_type(url: str) -> str: + if not url.startswith("data:") or ";base64," not in url: + return url + header, payload = url.split(";base64,", 1) + declared_mime_type = header[len("data:") :] + if declared_mime_type.lower() not in _GENERIC_DATA_URL_MIME_TYPES: + return url + sniffed_mime_type = _sniff_base64_image_mime_type(payload) + if sniffed_mime_type is None: + return url + return f"data:{sniffed_mime_type};base64,{payload}" + + +def _normalize_image_mime_type_in_content_item( + content_item: OpenAIMessageContentListBlock, +) -> OpenAIMessageContentListBlock: + if content_item["type"] != "image_url": + return content_item + image_url = content_item.get("image_url") + if isinstance(image_url, str): + normalized_item = content_item.copy() + normalized_item["image_url"] = normalize_image_data_url_mime_type(image_url) + return normalized_item + if not isinstance(image_url, dict) or "url" not in image_url: + return content_item + url = image_url["url"] + if not isinstance(url, str): + return content_item + normalized_image_url = image_url.copy() + normalized_image_url["url"] = normalize_image_data_url_mime_type(url) + normalized_item = content_item.copy() + normalized_item["image_url"] = normalized_image_url + return normalized_item + + +def _normalize_image_mime_types_in_message(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "user": + return message + content = message.get("content") + if not isinstance(content, list): + return message + normalized_message = message.copy() + normalized_message["content"] = [_normalize_image_mime_type_in_content_item(item) for item in content] + return normalized_message + + +def normalize_image_data_url_mime_types_in_messages( + messages: list[AllMessageValues], +) -> list[AllMessageValues]: + return [_normalize_image_mime_types_in_message(message) for message in messages] + + def get_tool_call_names(tools: List[ChatCompletionToolParam]) -> List[str]: """ Get tool call names from tools diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 31c913d5d4e..c66cf2b5015 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -7,6 +7,7 @@ from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overlo from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, + normalize_image_data_url_mime_types_in_messages, ) from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig @@ -45,6 +46,9 @@ def create_config_class(provider: SimpleProviderConfig): if provider.special_handling.get("convert_content_list_to_string"): messages = handle_messages_with_content_list_to_str_conversion(messages) + if provider.special_handling.get("normalize_image_mime_type"): + messages = normalize_image_data_url_mime_types_in_messages(messages) + if is_async: return super()._transform_messages(messages=messages, model=model, is_async=True) else: diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 164100d4194..711b9778410 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -173,7 +173,10 @@ "api_key_env": "META_API_KEY", "api_base_env": "META_API_BASE", "base_class": "openai_gpt", - "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"] + "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"], + "special_handling": { + "normalize_image_mime_type": true + } }, "pinstripes": { "base_url": "https://pinstripes.io/v1", diff --git a/tests/test_litellm/llms/openai_like/test_meta_provider.py b/tests/test_litellm/llms/openai_like/test_meta_provider.py index 11b78828da6..7769a5d8cc4 100644 --- a/tests/test_litellm/llms/openai_like/test_meta_provider.py +++ b/tests/test_litellm/llms/openai_like/test_meta_provider.py @@ -2,6 +2,9 @@ Tests for the Meta Model API (Muse Spark) provider configuration and integration. """ +import asyncio +import base64 + import litellm @@ -192,6 +195,101 @@ class TestMetaAnthropicMessages: assert headers["anthropic-version"] == "2023-06-01" +PNG_BASE64 = base64.b64encode(b"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a" + b"\x00" * 32).decode() +JPEG_BASE64 = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 32).decode() + + +def _transform_image_url_through_provider(url: str, provider: litellm.LlmProviders) -> str: + cfg = litellm.ProviderConfigManager.get_provider_chat_config(model="muse-spark-1.1", provider=provider) + assert cfg is not None + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + {"type": "image_url", "image_url": {"url": url}}, + ], + } + ] + out = cfg.transform_request( + model="muse-spark-1.1", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + return out["messages"][0]["content"][1]["image_url"]["url"] + + +class TestMetaImageMimeNormalization: + def test_meta_config_has_normalize_flag(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + meta = JSONProviderRegistry.get("meta") + assert meta is not None + assert meta.special_handling.get("normalize_image_mime_type") is True + + def test_octet_stream_png_rewritten(self): + url = _transform_image_url_through_provider( + f"data:binary/octet-stream;base64,{PNG_BASE64}", litellm.LlmProviders.META + ) + assert url == f"data:image/png;base64,{PNG_BASE64}" + + def test_application_octet_stream_jpeg_rewritten(self): + url = _transform_image_url_through_provider( + f"data:application/octet-stream;base64,{JPEG_BASE64}", litellm.LlmProviders.META + ) + assert url == f"data:image/jpeg;base64,{JPEG_BASE64}" + + def test_valid_mime_untouched(self): + original = f"data:image/jpeg;base64,{JPEG_BASE64}" + assert _transform_image_url_through_provider(original, litellm.LlmProviders.META) == original + + def test_mismatched_but_valid_mime_untouched(self): + original = f"data:image/jpeg;base64,{PNG_BASE64}" + assert _transform_image_url_through_provider(original, litellm.LlmProviders.META) == original + + def test_undecodable_payload_untouched(self): + original = "data:binary/octet-stream;base64,!!!not-base64!!!" + assert _transform_image_url_through_provider(original, litellm.LlmProviders.META) == original + + def test_http_url_untouched(self): + original = "https://example.com/image.png" + assert _transform_image_url_through_provider(original, litellm.LlmProviders.META) == original + + def test_provider_without_flag_untouched(self): + original = f"data:binary/octet-stream;base64,{PNG_BASE64}" + assert _transform_image_url_through_provider(original, litellm.LlmProviders.TENSORMESH) == original + + def test_async_transform_rewrites_octet_stream_png(self): + cfg = litellm.ProviderConfigManager.get_provider_chat_config( + model="muse-spark-1.1", provider=litellm.LlmProviders.META + ) + assert cfg is not None + messages = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": f"data:binary/octet-stream;base64,{PNG_BASE64}"}}], + } + ] + out = asyncio.run(cfg._transform_messages(messages=messages, model="muse-spark-1.1", is_async=True)) + assert out[0]["content"][0]["image_url"]["url"] == f"data:image/png;base64,{PNG_BASE64}" + + def test_string_content_passthrough(self): + cfg = litellm.ProviderConfigManager.get_provider_chat_config( + model="muse-spark-1.1", provider=litellm.LlmProviders.META + ) + assert cfg is not None + out = cfg.transform_request( + model="muse-spark-1.1", + messages=[{"role": "user", "content": "hello"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + assert out["messages"][0]["content"] == "hello" + + class TestMuseSparkModelInfo: def test_muse_spark_pricing_and_capabilities(self): info = litellm.get_model_info("meta/muse-spark-1.1")