diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 38a8856d8f4..e10fc6d85ea 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -898,25 +898,35 @@ def convert_to_anthropic_image_obj( else: media_type = media_type.replace("\\/", "/") - # Anthropic rejects requests where the declared media_type mismatches the actual - # image bytes. Auto-detect from magic bytes and override the declared type when needed. - try: - from litellm.litellm_core_utils.token_counter import get_image_type + # Anthropic rejects requests where the declared media_type mismatches the actual + # image bytes. Auto-detect from magic bytes and override only when the detected + # type is one Anthropic actually supports (jpeg/png/gif/webp). Falling back to + # "image/" + detected would produce "image/heic" for any ISO Base Media File + # Format container (MP4, MOV, …), making the failure mode worse than before. + # Skip detection when caller supplied an explicit `format` — that is authoritative. + _ANTHROPIC_SUPPORTED_MIME = { + "jpeg": "image/jpeg", + "png": "image/png", + "gif": "image/gif", + "webp": "image/webp", + } + try: + from litellm.litellm_core_utils.token_counter import get_image_type - image_bytes = base64.b64decode(base64_data[:128]) - detected = get_image_type(image_bytes) - if detected is not None: - detected_mime = "image/{}".format(detected) - if detected_mime != media_type: - verbose_logger.debug( - "convert_to_anthropic_image_obj: declared media_type=%s does not match " - "detected type=%s; using detected type.", - media_type, - detected_mime, - ) - media_type = detected_mime - except Exception: - pass + image_bytes = base64.b64decode(base64_data[:128]) + detected = get_image_type(image_bytes) + if detected is not None: + detected_mime = _ANTHROPIC_SUPPORTED_MIME.get(detected) + if detected_mime is not None and detected_mime != media_type: + verbose_logger.debug( + "convert_to_anthropic_image_obj: declared media_type=%s does not match " + "detected type=%s; using detected type.", + media_type, + detected_mime, + ) + media_type = detected_mime + except Exception: + pass return GenericImageParsingChunk( type="base64", diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 0566b04c6fd..62b7c7a8141 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -288,6 +288,10 @@ def _fix_image_media_types_in_messages(messages: List[Dict]) -> List[Dict]: from litellm import verbose_logger from litellm.litellm_core_utils.token_counter import get_image_type + # Only override with types Anthropic actually accepts. Using a fallback like + # "image/" + detected would produce "image/heic" for any ISO Base Media File + # Format container (MP4, MOV, …) since get_image_type() returns "heic" for + # all ftyp-box files — making the failure mode worse than before. _MIME_MAP = {"jpeg": "image/jpeg", "png": "image/png", "gif": "image/gif", "webp": "image/webp"} def _fix_source(source: dict) -> None: @@ -301,7 +305,10 @@ def _fix_image_media_types_in_messages(messages: List[Dict]) -> List[Dict]: detected = get_image_type(raw) if detected is None: return - detected_mime = _MIME_MAP.get(detected, "image/" + detected) + detected_mime = _MIME_MAP.get(detected) + if detected_mime is None: + # Detected type is not supported by Anthropic; keep declared type. + return declared = source.get("media_type", "") if detected_mime != declared: verbose_logger.debug( diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 10a3c107367..b32a9b02a0d 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3399,3 +3399,151 @@ def test_extract_response_content_thinking_block_null_thinking(): assert len(thinking_blocks) == 1 assert thinking_blocks[0]["thinking"] == "Let me think..." assert "Done" in text + + +_JPEG_B64 = "/9j/4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +_PNG_B64 = "iVBORw0KGgoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +_FTYP_B64 = "AAAAGGZ0eXBtcDQyAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + +class TestConvertToAnthropicImageObjMediaTypeDetection: + def test_jpeg_bytes_with_png_declaration_corrected(self): + with patch( + "litellm.litellm_core_utils.token_counter.get_image_type", + return_value="jpeg", + ): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_image_obj, + ) + + result = convert_to_anthropic_image_obj( + f"data:image/png;base64,{_JPEG_B64}", format=None + ) + assert result["media_type"] == "image/jpeg" + + def test_png_bytes_with_jpeg_declaration_corrected(self): + with patch( + "litellm.litellm_core_utils.token_counter.get_image_type", + return_value="png", + ): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_image_obj, + ) + + result = convert_to_anthropic_image_obj( + f"data:image/jpeg;base64,{_PNG_B64}", format=None + ) + assert result["media_type"] == "image/png" + + def test_correct_declaration_not_overridden(self): + with patch( + "litellm.litellm_core_utils.token_counter.get_image_type", + return_value="jpeg", + ): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_image_obj, + ) + + result = convert_to_anthropic_image_obj( + f"data:image/jpeg;base64,{_JPEG_B64}", format=None + ) + assert result["media_type"] == "image/jpeg" + + def test_heic_false_positive_keeps_declared_type(self): + with patch( + "litellm.litellm_core_utils.token_counter.get_image_type", + return_value="heic", + ): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_image_obj, + ) + + result = convert_to_anthropic_image_obj( + f"data:image/jpeg;base64,{_FTYP_B64}", format=None + ) + assert result["media_type"] == "image/jpeg" + + def test_explicit_format_not_overridden_by_detection(self): + with patch( + "litellm.litellm_core_utils.token_counter.get_image_type", + return_value="jpeg", + ): + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_anthropic_image_obj, + ) + + result = convert_to_anthropic_image_obj( + f"data:image/png;base64,{_JPEG_B64}", format="image/png" + ) + assert result["media_type"] == "image/png" + + +class TestFixImageMediaTypesInMessages: + def _make_messages(self, declared: str, b64_data: str) -> list: + return [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": declared, + "data": b64_data, + }, + } + ], + } + ] + + def test_jpeg_bytes_with_png_declaration_corrected(self): + with patch( + "litellm.litellm_core_utils.token_counter.get_image_type", + return_value="jpeg", + ): + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _fix_image_media_types_in_messages, + ) + + messages = self._make_messages("image/png", _JPEG_B64) + result = _fix_image_media_types_in_messages(messages) + assert result[0]["content"][0]["source"]["media_type"] == "image/jpeg" + + def test_correct_declaration_unchanged(self): + with patch( + "litellm.litellm_core_utils.token_counter.get_image_type", + return_value="jpeg", + ): + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _fix_image_media_types_in_messages, + ) + + messages = self._make_messages("image/jpeg", _JPEG_B64) + result = _fix_image_media_types_in_messages(messages) + assert result[0]["content"][0]["source"]["media_type"] == "image/jpeg" + + def test_heic_false_positive_keeps_declared_type(self): + with patch( + "litellm.litellm_core_utils.token_counter.get_image_type", + return_value="heic", + ): + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _fix_image_media_types_in_messages, + ) + + messages = self._make_messages("image/jpeg", _FTYP_B64) + result = _fix_image_media_types_in_messages(messages) + assert result[0]["content"][0]["source"]["media_type"] == "image/jpeg" + + def test_none_detection_keeps_declared_type(self): + with patch( + "litellm.litellm_core_utils.token_counter.get_image_type", + return_value=None, + ): + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _fix_image_media_types_in_messages, + ) + + messages = self._make_messages("image/png", _JPEG_B64) + result = _fix_image_media_types_in_messages(messages) + assert result[0]["content"][0]["source"]["media_type"] == "image/png"