Merge c75b596b84 into fe29c55d11

litellm/llms/ollama/chat/transformation.py

@@ -14,8 +14,13 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     convert_content_list_to_str,
     extract_images_from_message,
 )
+from litellm.litellm_core_utils.prompt_templates.image_handling import (
+    async_inline_remote_media,
+    inline_remote_image_urls,
+)
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
+from litellm.llms.ollama.common_utils import prepare_ollama_images
 from litellm.types.llms.ollama import (
     OllamaChatCompletionMessage,
     OllamaToolCall,
@@ -227,6 +232,26 @@ class OllamaChatConfig(BaseConfig):
 
         return url
 
+    @property
+    def uses_async_transform_request(self) -> bool:
+        return True
+
+    async def async_transform_request(
+        self,
+        model: str,
+        messages: list[AllMessageValues],  # mutable-ok: BaseConfig signature
+        optional_params: dict[str, object],  # mutable-ok: BaseConfig signature
+        litellm_params: dict[str, object],  # mutable-ok: BaseConfig signature
+        headers: dict[str, object],  # mutable-ok: BaseConfig signature
+    ) -> dict[str, object]:  # mutable-ok: BaseConfig signature
+        return self.transform_request(
+            model=model,
+            messages=await async_inline_remote_media(messages, should_inline=inline_remote_image_urls),
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
     def transform_request(
         self,
         model: str,
@@ -278,7 +303,7 @@ class OllamaChatConfig(BaseConfig):
             if content_str is not None:
                 ollama_message["content"] = content_str
             if images is not None:
-                ollama_message["images"] = images
+                ollama_message["images"] = prepare_ollama_images(images)
             if new_tools is not None:
                 ollama_message["tool_calls"] = new_tools
             tool_call_id = m.get("tool_call_id")

litellm/llms/ollama/common_utils.py

@@ -5,6 +5,12 @@ from typing import Any, Final
 import httpx
 
 from litellm import verbose_logger
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    _extract_base64_data,
+)
+from litellm.litellm_core_utils.prompt_templates.image_handling import (
+    convert_url_to_base64,
+)
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 
 
@@ -42,6 +48,21 @@ def _convert_image(image: str) -> str:
     return _reencode_as_jpeg(raw_image, original=image)
 
 
+def prepare_ollama_images(images: list[str]) -> list[str]:
+    """
+    Ensure each image is the pure base64 data Ollama expects.
+
+    Remote http(s) URLs are downloaded and base64-encoded, because Ollama
+    cannot fetch URLs itself and would otherwise try to base64-decode the raw
+    URL (issue #30313). Data URLs and raw base64 inputs are returned unchanged.
+    """
+    # convert_url_to_base64 returns a data URL; strip it to pure base64.
+    return [  # mutable-ok: OllamaChatCompletionMessage.images is a list[str]
+        _extract_base64_data(convert_url_to_base64(image)) if image.startswith(("http://", "https://")) else image
+        for image in images
+    ]
+
+
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
 
 

tests/unit/llms/ollama/test_ollama_chat_transformation.py

@@ -274,8 +274,13 @@ class TestOllamaChatConfigResponseFormat:
         assert result["messages"][0]["images"][0] == "image1data..."
         assert result["messages"][0]["images"][1] == "image2data..."
 
-    def test_transform_request_image_url_as_string(self):
-        """Test handling of image_url as direct string (edge case)"""
+    def test_transform_request_image_url_as_string(self, monkeypatch):
+        """Test handling of image_url as direct string (edge case). Remote URLs
+        are downloaded and base64-encoded for Ollama (issue #30313)."""
+        monkeypatch.setattr(
+            "litellm.llms.ollama.common_utils.convert_url_to_base64",
+            lambda url: "data:image/jpeg;base64,downloadedbase64",
+        )
         config = OllamaChatConfig()
 
         # Test message with image_url as string (edge case from extract_images_from_message)
@@ -303,10 +308,10 @@ class TestOllamaChatConfigResponseFormat:
             headers={},
         )
 
-        # Verify image URL was extracted
+        # Verify the remote image URL was downloaded and base64-encoded
         assert "images" in result["messages"][0]
         assert len(result["messages"][0]["images"]) == 1
-        assert result["messages"][0]["images"][0] == "https://example.com/image.jpg"
+        assert result["messages"][0]["images"][0] == "downloadedbase64"
 
     def test_transform_request_no_images_no_images_key(self):
         """Test that messages without images don't have images key"""
@@ -989,3 +994,83 @@ class TestOllamaStreamingUsage:
         )
 
         assert result.usage is None
+
+
+def test_prepare_ollama_images_downloads_urls_and_passes_through_base64(monkeypatch):
+    """prepare_ollama_images downloads remote http(s) URLs (issue #30313) and
+    leaves already-base64 data untouched."""
+    from litellm.llms.ollama.common_utils import prepare_ollama_images
+
+    monkeypatch.setattr(
+        "litellm.llms.ollama.common_utils.convert_url_to_base64",
+        lambda url: "data:image/png;base64,downloaded",
+    )
+
+    result = prepare_ollama_images(
+        ["alreadybase64data", "https://example.com/image.png"]
+    )
+
+    assert result == ["alreadybase64data", "downloaded"]
+
+
+_REMOTE_IMAGE_MESSAGES = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "What's in this image?"},
+            {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}},
+        ],
+    }
+]
+
+
+@pytest.mark.asyncio
+async def test_async_transform_request_downloads_remote_images_without_blocking(monkeypatch):
+    """acompletion fetches remote images with async_convert_url_to_base64
+    instead of the blocking convert_url_to_base64."""
+
+    async def fake_async_convert_url_to_base64(url):
+        return "data:image/png;base64,downloaded"
+
+    def blocking_fetch(url):
+        raise AssertionError("the async path must not use the blocking fetch")
+
+    monkeypatch.setattr(
+        "litellm.litellm_core_utils.prompt_templates.image_handling.async_convert_url_to_base64",
+        fake_async_convert_url_to_base64,
+    )
+    monkeypatch.setattr("litellm.llms.ollama.common_utils.convert_url_to_base64", blocking_fetch)
+    config = OllamaChatConfig()
+
+    result = await config.async_transform_request(
+        model="llava",
+        messages=_REMOTE_IMAGE_MESSAGES,
+        optional_params={},
+        litellm_params={},
+        headers={},
+    )
+
+    assert config.uses_async_transform_request is True
+    assert result["messages"][0]["images"] == ["downloaded"]
+
+
+@pytest.mark.asyncio
+async def test_remote_image_download_respects_max_image_url_download_size_gate(monkeypatch):
+    """MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0 turns remote image downloads off on both
+    the sync and async ollama_chat paths, before any request goes out."""
+    monkeypatch.setattr(
+        "litellm.litellm_core_utils.prompt_templates.image_handling.MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 0
+    )
+    config = OllamaChatConfig()
+    request = {
+        "model": "llava",
+        "messages": _REMOTE_IMAGE_MESSAGES,
+        "optional_params": {},
+        "litellm_params": {},
+        "headers": {},
+    }
+
+    with pytest.raises(litellm.ImageFetchError, match="download is disabled"):
+        config.transform_request(**request)
+    with pytest.raises(litellm.ImageFetchError, match="download is disabled"):
+        await config.async_transform_request(**request)