diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index c44c80bc0a0..cb5c02ce10e 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -310,6 +310,26 @@ async def _fetch_data_urls(remote_urls: tuple[str, ...]) -> tuple[str, ...]: raise +def inline_remote_media( + messages: list[AllMessageValues], # mutable-ok: every transform_request takes list[AllMessageValues] + should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url, +) -> list[AllMessageValues]: # mutable-ok: every transform_request takes list[AllMessageValues] + remote_urls: Final = tuple( + dict.fromkeys( + remote.url + for message in messages + for part in _content_parts(message) + if (remote := _parse_remote_part(part)) is not None and should_inline(_remote_media(remote)) + ) + ) + if not remote_urls: + return messages + data_urls: Final = MappingProxyType({url: convert_url_to_base64(url) for url in remote_urls}) + return [ # mutable-ok: transform_request takes a list + _inline_message(message, data_urls, should_inline) for message in messages + ] + + async def async_inline_remote_media( messages: list[AllMessageValues], # mutable-ok: every transform_request takes list[AllMessageValues] should_inline: Callable[[RemoteMedia], bool] = inline_every_remote_url, diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 123531cf227..d907ef613a2 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -24,8 +24,8 @@ from typing_extensions import assert_never import litellm from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_inline_remote_media, - convert_url_to_base64, inline_remote_image_urls, + inline_remote_media, ) from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM @@ -172,48 +172,6 @@ def split_reasoning_tag(content: str) -> tuple[str | None, str]: return reasoning or None, body -def _remote_http_url(candidate: object) -> str | None: - return candidate if isinstance(candidate, str) and candidate.startswith(("http://", "https://")) else None - - -def _inlined_image_url_part(part: object) -> object: - fields: Final = part if isinstance(part, Mapping) else None - if fields is None or fields.get("type") != "image_url": - return part - image_url: Final = fields.get("image_url") - image_url_fields: Final = image_url if isinstance(image_url, Mapping) else None - url: Final = _remote_http_url(image_url_fields.get("url") if image_url_fields is not None else image_url) - if url is None: - return part - data_url: Final = convert_url_to_base64(url) - inlined: Final = {**image_url_fields, "url": data_url} if image_url_fields is not None else data_url - return {**fields, "image_url": inlined} # mutable-ok: json-serialized message part - - -def _inlined_image_url_message(message: AllMessageValues) -> AllMessageValues: - content: Final = message.get("content") - if not isinstance(content, list): - return message - inlined_message: Final = { # mutable-ok: json-serialized message - **message, - "content": [_inlined_image_url_part(part) for part in content], - } - return inlined_message # pyright: ignore[reportReturnType] # the same message with remote image parts inlined - - -def _with_inlined_remote_image_urls( - messages: list[AllMessageValues], -) -> list[AllMessageValues]: # mutable-ok: transform_request takes a list - """Inline every remote ``image_url`` so AWS never sees the ``http(s)://`` URLs it rejects. - - AWS's native surface only takes inline ``data:`` URLs and S3 URLs where Converse downloaded - remote images itself, so the bytes are fetched and inlined here exactly like Converse did. - """ - return [ # mutable-ok: transform_request takes a list - _inlined_image_url_message(message) for message in messages - ] - - class BedrockRuntimeChatCompletionsStreamingHandler(OpenAIChatCompletionStreamingHandler): """OpenAI chunk parsing plus the ```` split, tracked per choice index.""" @@ -376,7 +334,7 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): ) -> dict: # mutable-ok: BaseConfig signature return super().transform_request( model=split_bedrock_region_path(model)[1], - messages=_with_inlined_remote_image_urls(messages), + messages=inline_remote_media(messages, should_inline=inline_remote_image_urls), optional_params=self._inference_params(optional_params), litellm_params=litellm_params, headers=headers, diff --git a/tests/unit/litellm_core_utils/test_image_handling.py b/tests/unit/litellm_core_utils/test_image_handling.py index 21e97e97357..1fc8d54cecc 100644 --- a/tests/unit/litellm_core_utils/test_image_handling.py +++ b/tests/unit/litellm_core_utils/test_image_handling.py @@ -16,6 +16,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, async_inline_remote_media, convert_url_to_base64, + inline_remote_media, ) from litellm.litellm_core_utils.url_utils import SSRFError @@ -320,6 +321,50 @@ async def test_async_inline_remote_media_inlines_every_remote_part_shape(async_o assert messages == snapshot +def test_inline_remote_media_inlines_every_remote_part_shape(monkeypatch): + image_url = f"http://img.example/{uuid.uuid4()}.png" + pdf_url = f"http://docs.example/{uuid.uuid4()}.pdf" + fetched = [] + + def fake_convert(url): + fetched.append(url) + return f"data:image/png;base64,{url}" + + monkeypatch.setattr(image_handling, "convert_url_to_base64", fake_convert) + messages = [ + {"role": "system", "content": "be terse"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this?"}, + {"type": "image_url", "image_url": {"url": image_url, "detail": "low"}}, + {"type": "image_url", "image_url": image_url}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}, + {"type": "image_url", "image_url": {"url": "s3://bucket/key.png"}}, + {"type": "file", "file": {"file_id": pdf_url}}, + {"type": "document", "source": {"type": "url", "url": pdf_url}, "title": "the doc"}, + ], + }, + ] + snapshot = copy.deepcopy(messages) + + inlined = inline_remote_media(messages, should_inline=image_handling.inline_remote_image_urls) + + data_url = f"data:image/png;base64,{image_url}" + assert inlined[0] == {"role": "system", "content": "be terse"} + assert inlined[1]["content"] == [ + {"type": "text", "text": "what is this?"}, + {"type": "image_url", "image_url": {"url": data_url, "detail": "low"}}, + {"type": "image_url", "image_url": data_url}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}, + {"type": "image_url", "image_url": {"url": "s3://bucket/key.png"}}, + {"type": "file", "file": {"file_id": pdf_url}}, + {"type": "document", "source": {"type": "url", "url": pdf_url}, "title": "the doc"}, + ] + assert fetched == [image_url] + assert messages == snapshot + + async def test_async_inline_remote_media_inlines_only_the_parts_the_predicate_accepts(async_only_image_fetch): files_api_prefix = "https://generativelanguage.googleapis.com/v1beta/files/" files_api_pdf = f"{files_api_prefix}{uuid.uuid4().hex}" diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 4bfa7bf0fc4..cbde3dd03a0 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -360,10 +360,10 @@ def _assert_remote_images_inlined(content): def test_transform_request_inlines_remote_image_urls(local_cost_map, monkeypatch): - import litellm.llms.bedrock.chat.chat_completions.transformation as native_cc + import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling monkeypatch.setattr( - native_cc, "convert_url_to_base64", lambda url: f"data:image/png;base64,{url}" + image_handling, "convert_url_to_base64", lambda url: f"data:image/png;base64,{url}" ) body = AmazonBedrockRuntimeChatCompletionsConfig().transform_request( model="us.xai.grok-4.6",