fix(anthropic): guard magic-byte detection against HEIC false-positives

- factory.py: skip detection when explicit `format` param is supplied
  (format is authoritative, magic bytes should not override it)
- factory.py + handler.py: only override declared media_type when the
  detected type is in Anthropic's accepted set (jpeg/png/gif/webp);
  previously 'image/' + detected would produce 'image/heic' for any
  ISO Base Media File Format container (MP4/MOV/M4A), making the
  failure mode worse than the original mismatch
- Add 9 unit tests covering: mismatch correction, correct declarations
  unchanged, HEIC false-positive safety, explicit format precedence,
  and None detection passthrough

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -898,25 +898,35 @@ def convert_to_anthropic_image_obj(
         else:
             media_type = media_type.replace("\\/", "/")
 
-        # Anthropic rejects requests where the declared media_type mismatches the actual
-        # image bytes. Auto-detect from magic bytes and override the declared type when needed.
-        try:
-            from litellm.litellm_core_utils.token_counter import get_image_type
+            # Anthropic rejects requests where the declared media_type mismatches the actual
+            # image bytes. Auto-detect from magic bytes and override only when the detected
+            # type is one Anthropic actually supports (jpeg/png/gif/webp). Falling back to
+            # "image/" + detected would produce "image/heic" for any ISO Base Media File
+            # Format container (MP4, MOV, …), making the failure mode worse than before.
+            # Skip detection when caller supplied an explicit `format` — that is authoritative.
+            _ANTHROPIC_SUPPORTED_MIME = {
+                "jpeg": "image/jpeg",
+                "png": "image/png",
+                "gif": "image/gif",
+                "webp": "image/webp",
+            }
+            try:
+                from litellm.litellm_core_utils.token_counter import get_image_type
 
-            image_bytes = base64.b64decode(base64_data[:128])
-            detected = get_image_type(image_bytes)
-            if detected is not None:
-                detected_mime = "image/{}".format(detected)
-                if detected_mime != media_type:
-                    verbose_logger.debug(
-                        "convert_to_anthropic_image_obj: declared media_type=%s does not match "
-                        "detected type=%s; using detected type.",
-                        media_type,
-                        detected_mime,
-                    )
-                    media_type = detected_mime
-        except Exception:
-            pass
+                image_bytes = base64.b64decode(base64_data[:128])
+                detected = get_image_type(image_bytes)
+                if detected is not None:
+                    detected_mime = _ANTHROPIC_SUPPORTED_MIME.get(detected)
+                    if detected_mime is not None and detected_mime != media_type:
+                        verbose_logger.debug(
+                            "convert_to_anthropic_image_obj: declared media_type=%s does not match "
+                            "detected type=%s; using detected type.",
+                            media_type,
+                            detected_mime,
+                        )
+                        media_type = detected_mime
+            except Exception:
+                pass
 
         return GenericImageParsingChunk(
             type="base64",

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -288,6 +288,10 @@ def _fix_image_media_types_in_messages(messages: List[Dict]) -> List[Dict]:
     from litellm import verbose_logger
     from litellm.litellm_core_utils.token_counter import get_image_type
 
+    # Only override with types Anthropic actually accepts. Using a fallback like
+    # "image/" + detected would produce "image/heic" for any ISO Base Media File
+    # Format container (MP4, MOV, …) since get_image_type() returns "heic" for
+    # all ftyp-box files — making the failure mode worse than before.
     _MIME_MAP = {"jpeg": "image/jpeg", "png": "image/png", "gif": "image/gif", "webp": "image/webp"}
 
     def _fix_source(source: dict) -> None:
@@ -301,7 +305,10 @@ def _fix_image_media_types_in_messages(messages: List[Dict]) -> List[Dict]:
             detected = get_image_type(raw)
             if detected is None:
                 return
-            detected_mime = _MIME_MAP.get(detected, "image/" + detected)
+            detected_mime = _MIME_MAP.get(detected)
+            if detected_mime is None:
+                # Detected type is not supported by Anthropic; keep declared type.
+                return
             declared = source.get("media_type", "")
             if detected_mime != declared:
                 verbose_logger.debug(

tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py

@@ -3399,3 +3399,151 @@ def test_extract_response_content_thinking_block_null_thinking():
     assert len(thinking_blocks) == 1
     assert thinking_blocks[0]["thinking"] == "Let me think..."
     assert "Done" in text
+
+
+_JPEG_B64 = "/9j/4AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+_PNG_B64 = "iVBORw0KGgoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+_FTYP_B64 = "AAAAGGZ0eXBtcDQyAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+
+
+class TestConvertToAnthropicImageObjMediaTypeDetection:
+    def test_jpeg_bytes_with_png_declaration_corrected(self):
+        with patch(
+            "litellm.litellm_core_utils.token_counter.get_image_type",
+            return_value="jpeg",
+        ):
+            from litellm.litellm_core_utils.prompt_templates.factory import (
+                convert_to_anthropic_image_obj,
+            )
+
+            result = convert_to_anthropic_image_obj(
+                f"data:image/png;base64,{_JPEG_B64}", format=None
+            )
+            assert result["media_type"] == "image/jpeg"
+
+    def test_png_bytes_with_jpeg_declaration_corrected(self):
+        with patch(
+            "litellm.litellm_core_utils.token_counter.get_image_type",
+            return_value="png",
+        ):
+            from litellm.litellm_core_utils.prompt_templates.factory import (
+                convert_to_anthropic_image_obj,
+            )
+
+            result = convert_to_anthropic_image_obj(
+                f"data:image/jpeg;base64,{_PNG_B64}", format=None
+            )
+            assert result["media_type"] == "image/png"
+
+    def test_correct_declaration_not_overridden(self):
+        with patch(
+            "litellm.litellm_core_utils.token_counter.get_image_type",
+            return_value="jpeg",
+        ):
+            from litellm.litellm_core_utils.prompt_templates.factory import (
+                convert_to_anthropic_image_obj,
+            )
+
+            result = convert_to_anthropic_image_obj(
+                f"data:image/jpeg;base64,{_JPEG_B64}", format=None
+            )
+            assert result["media_type"] == "image/jpeg"
+
+    def test_heic_false_positive_keeps_declared_type(self):
+        with patch(
+            "litellm.litellm_core_utils.token_counter.get_image_type",
+            return_value="heic",
+        ):
+            from litellm.litellm_core_utils.prompt_templates.factory import (
+                convert_to_anthropic_image_obj,
+            )
+
+            result = convert_to_anthropic_image_obj(
+                f"data:image/jpeg;base64,{_FTYP_B64}", format=None
+            )
+            assert result["media_type"] == "image/jpeg"
+
+    def test_explicit_format_not_overridden_by_detection(self):
+        with patch(
+            "litellm.litellm_core_utils.token_counter.get_image_type",
+            return_value="jpeg",
+        ):
+            from litellm.litellm_core_utils.prompt_templates.factory import (
+                convert_to_anthropic_image_obj,
+            )
+
+            result = convert_to_anthropic_image_obj(
+                f"data:image/png;base64,{_JPEG_B64}", format="image/png"
+            )
+            assert result["media_type"] == "image/png"
+
+
+class TestFixImageMediaTypesInMessages:
+    def _make_messages(self, declared: str, b64_data: str) -> list:
+        return [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image",
+                        "source": {
+                            "type": "base64",
+                            "media_type": declared,
+                            "data": b64_data,
+                        },
+                    }
+                ],
+            }
+        ]
+
+    def test_jpeg_bytes_with_png_declaration_corrected(self):
+        with patch(
+            "litellm.litellm_core_utils.token_counter.get_image_type",
+            return_value="jpeg",
+        ):
+            from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
+                _fix_image_media_types_in_messages,
+            )
+
+            messages = self._make_messages("image/png", _JPEG_B64)
+            result = _fix_image_media_types_in_messages(messages)
+            assert result[0]["content"][0]["source"]["media_type"] == "image/jpeg"
+
+    def test_correct_declaration_unchanged(self):
+        with patch(
+            "litellm.litellm_core_utils.token_counter.get_image_type",
+            return_value="jpeg",
+        ):
+            from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
+                _fix_image_media_types_in_messages,
+            )
+
+            messages = self._make_messages("image/jpeg", _JPEG_B64)
+            result = _fix_image_media_types_in_messages(messages)
+            assert result[0]["content"][0]["source"]["media_type"] == "image/jpeg"
+
+    def test_heic_false_positive_keeps_declared_type(self):
+        with patch(
+            "litellm.litellm_core_utils.token_counter.get_image_type",
+            return_value="heic",
+        ):
+            from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
+                _fix_image_media_types_in_messages,
+            )
+
+            messages = self._make_messages("image/jpeg", _FTYP_B64)
+            result = _fix_image_media_types_in_messages(messages)
+            assert result[0]["content"][0]["source"]["media_type"] == "image/jpeg"
+
+    def test_none_detection_keeps_declared_type(self):
+        with patch(
+            "litellm.litellm_core_utils.token_counter.get_image_type",
+            return_value=None,
+        ):
+            from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
+                _fix_image_media_types_in_messages,
+            )
+
+            messages = self._make_messages("image/png", _JPEG_B64)
+            result = _fix_image_media_types_in_messages(messages)
+            assert result[0]["content"][0]["source"]["media_type"] == "image/png"