From 4fd719659e23a18eba540e8619f9fc1d0a02933c Mon Sep 17 00:00:00 2001 From: Rodrigo-Palma Date: Wed, 1 Jul 2026 20:08:43 -0300 Subject: [PATCH 1/2] fix(ollama): download remote image URLs for ollama_chat instead of sending them verbatim ollama_chat forwarded remote http(s) image_url values straight into Ollama's `images` field, so Ollama tried to base64-decode the URL and failed with "illegal base64 data at input byte 5". Ollama does not accept URLs. The ollama_chat transformation now downloads and base64-encodes remote http(s) URLs via a new `prepare_ollama_images` helper in the ollama provider utils (reusing convert_url_to_base64), matching the ollama completion (/api/generate) path. Data URLs and raw base64 inputs are unchanged. Fixes #30313 --- litellm/llms/ollama/chat/transformation.py | 3 +- litellm/llms/ollama/common_utils.py | 21 +++++++++++++ .../ollama/test_ollama_chat_transformation.py | 30 ++++++++++++++++--- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index cb3080e6534..3f151628b6d 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -16,6 +16,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.ollama.common_utils import prepare_ollama_images from litellm.types.llms.ollama import ( OllamaChatCompletionMessage, OllamaToolCall, @@ -278,7 +279,7 @@ class OllamaChatConfig(BaseConfig): if content_str is not None: ollama_message["content"] = content_str if images is not None: - ollama_message["images"] = images + ollama_message["images"] = prepare_ollama_images(images) if new_tools is not None: ollama_message["tool_calls"] = new_tools tool_call_id = m.get("tool_call_id") diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 9f46cbc5cd5..36ffb6a4847 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -5,6 +5,12 @@ from typing import Any, Final import httpx from litellm import verbose_logger +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _extract_base64_data, +) +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + convert_url_to_base64, +) from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -42,6 +48,21 @@ def _convert_image(image: str) -> str: return _reencode_as_jpeg(raw_image, original=image) +def prepare_ollama_images(images: list[str]) -> list[str]: + """ + Ensure each image is the pure base64 data Ollama expects. + + Remote http(s) URLs are downloaded and base64-encoded, because Ollama + cannot fetch URLs itself and would otherwise try to base64-decode the raw + URL (issue #30313). Data URLs and raw base64 inputs are returned unchanged. + """ + # convert_url_to_base64 returns a data URL; strip it to pure base64. + return [ # mutable-ok: OllamaChatCompletionMessage.images is a list[str] + _extract_base64_data(convert_url_to_base64(image)) if image.startswith(("http://", "https://")) else image + for image in images + ] + + from litellm.llms.base_llm.base_utils import BaseLLMModelInfo diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index c7fba21d222..8bf26e45160 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -274,8 +274,13 @@ class TestOllamaChatConfigResponseFormat: assert result["messages"][0]["images"][0] == "image1data..." assert result["messages"][0]["images"][1] == "image2data..." - def test_transform_request_image_url_as_string(self): - """Test handling of image_url as direct string (edge case)""" + def test_transform_request_image_url_as_string(self, monkeypatch): + """Test handling of image_url as direct string (edge case). Remote URLs + are downloaded and base64-encoded for Ollama (issue #30313).""" + monkeypatch.setattr( + "litellm.llms.ollama.common_utils.convert_url_to_base64", + lambda url: "data:image/jpeg;base64,downloadedbase64", + ) config = OllamaChatConfig() # Test message with image_url as string (edge case from extract_images_from_message) @@ -303,10 +308,10 @@ class TestOllamaChatConfigResponseFormat: headers={}, ) - # Verify image URL was extracted + # Verify the remote image URL was downloaded and base64-encoded assert "images" in result["messages"][0] assert len(result["messages"][0]["images"]) == 1 - assert result["messages"][0]["images"][0] == "https://example.com/image.jpg" + assert result["messages"][0]["images"][0] == "downloadedbase64" def test_transform_request_no_images_no_images_key(self): """Test that messages without images don't have images key""" @@ -989,3 +994,20 @@ class TestOllamaStreamingUsage: ) assert result.usage is None + + +def test_prepare_ollama_images_downloads_urls_and_passes_through_base64(monkeypatch): + """prepare_ollama_images downloads remote http(s) URLs (issue #30313) and + leaves already-base64 data untouched.""" + from litellm.llms.ollama.common_utils import prepare_ollama_images + + monkeypatch.setattr( + "litellm.llms.ollama.common_utils.convert_url_to_base64", + lambda url: "data:image/png;base64,downloaded", + ) + + result = prepare_ollama_images( + ["alreadybase64data", "https://example.com/image.png"] + ) + + assert result == ["alreadybase64data", "downloaded"] From c75b596b84392efc82cecbb69d1b4f2834ced04c Mon Sep 17 00:00:00 2001 From: Rodrigo-Palma Date: Thu, 1 Oct 2026 09:27:15 -0300 Subject: [PATCH 2/2] fix(ollama): fetch remote images asynchronously on ollama_chat acompletion Override async_transform_request so acompletion inlines remote image_url parts with async_inline_remote_media, the same path the ollama completion config uses, instead of the blocking convert_url_to_base64. The sync path is kept for completion(). Both paths keep honouring MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, where 0 turns remote downloads off. --- litellm/llms/ollama/chat/transformation.py | 24 +++++++ .../ollama/test_ollama_chat_transformation.py | 63 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 3f151628b6d..a4e36c40b77 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -14,6 +14,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_images_from_message, ) +from litellm.litellm_core_utils.prompt_templates.image_handling import ( + async_inline_remote_media, + inline_remote_image_urls, +) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.ollama.common_utils import prepare_ollama_images @@ -228,6 +232,26 @@ class OllamaChatConfig(BaseConfig): return url + @property + def uses_async_transform_request(self) -> bool: + return True + + async def async_transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict[str, object], # mutable-ok: BaseConfig signature + litellm_params: dict[str, object], # mutable-ok: BaseConfig signature + headers: dict[str, object], # mutable-ok: BaseConfig signature + ) -> dict[str, object]: # mutable-ok: BaseConfig signature + return self.transform_request( + model=model, + messages=await async_inline_remote_media(messages, should_inline=inline_remote_image_urls), + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + def transform_request( self, model: str, diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index 8bf26e45160..afce1642af6 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -1011,3 +1011,66 @@ def test_prepare_ollama_images_downloads_urls_and_passes_through_base64(monkeypa ) assert result == ["alreadybase64data", "downloaded"] + + +_REMOTE_IMAGE_MESSAGES = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}, + ], + } +] + + +@pytest.mark.asyncio +async def test_async_transform_request_downloads_remote_images_without_blocking(monkeypatch): + """acompletion fetches remote images with async_convert_url_to_base64 + instead of the blocking convert_url_to_base64.""" + + async def fake_async_convert_url_to_base64(url): + return "data:image/png;base64,downloaded" + + def blocking_fetch(url): + raise AssertionError("the async path must not use the blocking fetch") + + monkeypatch.setattr( + "litellm.litellm_core_utils.prompt_templates.image_handling.async_convert_url_to_base64", + fake_async_convert_url_to_base64, + ) + monkeypatch.setattr("litellm.llms.ollama.common_utils.convert_url_to_base64", blocking_fetch) + config = OllamaChatConfig() + + result = await config.async_transform_request( + model="llava", + messages=_REMOTE_IMAGE_MESSAGES, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert config.uses_async_transform_request is True + assert result["messages"][0]["images"] == ["downloaded"] + + +@pytest.mark.asyncio +async def test_remote_image_download_respects_max_image_url_download_size_gate(monkeypatch): + """MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0 turns remote image downloads off on both + the sync and async ollama_chat paths, before any request goes out.""" + monkeypatch.setattr( + "litellm.litellm_core_utils.prompt_templates.image_handling.MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 0 + ) + config = OllamaChatConfig() + request = { + "model": "llava", + "messages": _REMOTE_IMAGE_MESSAGES, + "optional_params": {}, + "litellm_params": {}, + "headers": {}, + } + + with pytest.raises(litellm.ImageFetchError, match="download is disabled"): + config.transform_request(**request) + with pytest.raises(litellm.ImageFetchError, match="download is disabled"): + await config.async_transform_request(**request)