fix(ollama): send PNG and JPEG images without requiring Pillow. (#41979)

* fix(ollama): send PNG and JPEG images without requiring Pillow

The ollama/ completion transport imported Pillow before it looked at the image, so every image request failed with a 500 on installs without Pillow. That includes the Docker image, where Pillow is only a CI dependency

Detect PNG and JPEG from their leading bytes and pass them through untouched. Pillow is now imported only when another format has to be re-encoded as JPEG, and that case still raises the same install hint

* fix(ollama): address Greptile findings on image conversion

Catch all exceptions on Pillow import, not just ImportError, so the helpful
install hint always appears. Break a line that exceeded 120 characters

litellm/llms/ollama/common_utils.py

@@ -1,3 +1,5 @@
+import base64
+import io
 from typing import Any, Final
 
 import httpx
@@ -11,37 +13,35 @@ class OllamaError(BaseLLMException):
         super().__init__(status_code=status_code, message=message, headers=headers)
 
 
-def _convert_image(image):
-    """
-    Convert image to base64 encoded image if not already in base64 format
+_JPEG_AND_PNG_SIGNATURES: Final = (b"\xff\xd8\xff", b"\x89PNG\r\n\x1a\n")
 
-    If image is already in base64 format AND is a jpeg/png, return it
-
-    If image is not JPEG/PNG, convert it to JPEG base64 format
-    """
-    import base64
-    import io
 
+def _reencode_as_jpeg(raw_image: bytes, original: str) -> str:
     try:
         from PIL import Image
     except Exception:
         raise Exception("ollama image conversion failed please run `pip install Pillow`")
 
-    orig: Final = image
-    if image.startswith("data:"):
-        image = image.split(",")[-1]
     try:
-        image_data: Final = Image.open(io.BytesIO(base64.b64decode(image)))
-        if image_data.format in ["JPEG", "PNG"]:
-            return image
+        picture: Final = Image.open(io.BytesIO(raw_image))
     except Exception:
-        return orig
+        return original
     jpeg_image: Final = io.BytesIO()
-    image_data.convert("RGB").save(jpeg_image, "JPEG")
-    jpeg_image.seek(0)
+    picture.convert("RGB").save(jpeg_image, "JPEG")
     return base64.b64encode(jpeg_image.getvalue()).decode("utf-8")
 
 
+def _convert_image(image: str) -> str:
+    payload: Final = image.split(",")[-1] if image.startswith("data:") else image
+    try:
+        raw_image: Final = base64.b64decode(payload)
+    except ValueError:
+        return image
+    if raw_image.startswith(_JPEG_AND_PNG_SIGNATURES):
+        return payload
+    return _reencode_as_jpeg(raw_image, original=image)
+
+
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
 
 

tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py

@@ -1,4 +1,7 @@
+import base64
+import io
 import json
+import sys
 from litellm._uuid import uuid
 from unittest.mock import MagicMock, patch
 
@@ -544,3 +547,74 @@ async def test_ollama_async_completion_inlines_remote_images_off_the_event_loop(
     assert response.choices[0].message.content == "Green"
     assert async_only_image_fetch.fetched == [image_url]
     assert captured["body"]["images"] == [async_only_image_fetch.base64_png]
+
+
+def _image_base64(image_format: str) -> str:
+    from PIL import Image
+
+    buffer = io.BytesIO()
+    Image.new("RGB", (4, 4), "green").save(buffer, image_format)
+    return base64.b64encode(buffer.getvalue()).decode("utf-8")
+
+
+def _transform_image_request(image_base64: str, mime_subtype: str) -> dict:
+    return OllamaConfig().transform_request(
+        model="llava",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "What colour is this?"},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/{mime_subtype};base64,{image_base64}"},
+                    },
+                ],
+            }
+        ],
+        optional_params={},
+        litellm_params={},
+        headers={},
+    )
+
+
+@pytest.mark.parametrize("image_format", ["PNG", "JPEG"])
+def test_transform_request_sends_png_and_jpeg_images_without_pillow(
+    image_format: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    image_base64 = _image_base64(image_format)
+    monkeypatch.setitem(sys.modules, "PIL", None)
+
+    data = _transform_image_request(image_base64, image_format.lower())
+
+    assert data["images"] == [image_base64]
+
+
+def test_transform_request_without_pillow_says_how_to_convert_other_image_formats(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    gif_base64 = _image_base64("GIF")
+    monkeypatch.setitem(sys.modules, "PIL", None)
+
+    with pytest.raises(Exception, match="pip install Pillow"):
+        _transform_image_request(gif_base64, "gif")
+
+
+def test_transform_request_reencodes_other_image_formats_as_jpeg() -> None:
+    from PIL import Image
+
+    data = _transform_image_request(_image_base64("GIF"), "gif")
+
+    (encoded,) = data["images"]
+    assert Image.open(io.BytesIO(base64.b64decode(encoded))).format == "JPEG"
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [base64.b64encode(b"not an image").decode("utf-8"), "abc"],
+    ids=["decodable_but_not_an_image", "invalid_base64"],
+)
+def test_transform_request_leaves_unreadable_images_untouched(payload: str) -> None:
+    data = _transform_image_request(payload, "png")
+
+    assert data["images"] == [payload]