From b8aa8dea1a0e681c858ab9f471cdb1f33936cb0e Mon Sep 17 00:00:00 2001 From: Ryan H <3118399+ryanh-ai@users.noreply.github.com> Date: Sat, 14 Mar 2026 19:50:22 -0700 Subject: [PATCH] feat(bedrock): S3 URL pass-through for multimodal content blocks --- .../prompt_templates/factory.py | 176 ++++++- ...odel_prices_and_context_window_backup.json | 83 ++-- litellm/types/llms/bedrock.py | 8 +- litellm/types/utils.py | 1 + litellm/utils.py | 7 + model_prices_and_context_window.json | 83 ++-- .../local_testing/test_bedrock_s3_content.py | 183 +++++++ .../bedrock/chat/test_s3_content_blocks.py | 468 ++++++++++++++++++ tests/test_litellm/test_utils.py | 1 + 9 files changed, 918 insertions(+), 92 deletions(-) create mode 100644 tests/local_testing/test_bedrock_s3_content.py create mode 100644 tests/test_litellm/llms/bedrock/chat/test_s3_content_blocks.py diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index f6004616712..12254012499 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -3,6 +3,7 @@ import copy import hashlib import json import mimetypes +import os import re import xml.etree.ElementTree as ET from enum import Enum @@ -33,6 +34,7 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallFunctionChunk, ChatCompletionToolMessage, ChatCompletionUserMessage, + ChatCompletionVideoObject, OpenAIMessageContentListBlock, ) from litellm.types.llms.vertex_ai import FunctionCall as VertexFunctionCall @@ -3468,6 +3470,7 @@ from litellm.types.llms.bedrock import ( from litellm.types.llms.bedrock import ContentBlock as BedrockContentBlock from litellm.types.llms.bedrock import DocumentBlock as BedrockDocumentBlock from litellm.types.llms.bedrock import ImageBlock as BedrockImageBlock +from litellm.types.llms.bedrock import S3Location as BedrockS3Location from litellm.types.llms.bedrock import SourceBlock as BedrockSourceBlock from litellm.types.llms.bedrock import ToolBlock as BedrockToolBlock from litellm.types.llms.bedrock import ( @@ -3481,6 +3484,7 @@ from litellm.types.llms.bedrock import ( from litellm.types.llms.bedrock import ToolSpecBlock as BedrockToolSpecBlock from litellm.types.llms.bedrock import ToolUseBlock as BedrockToolUseBlock from litellm.types.llms.bedrock import VideoBlock as BedrockVideoBlock +from litellm.utils import supports_s3_input def _parse_content_type(content_type: str) -> str: @@ -3498,7 +3502,14 @@ def _parse_mime_type(base64_data: str) -> Optional[str]: class BedrockImageProcessor: - """Handles both sync and async image processing for Bedrock conversations.""" + """Handles both sync and async image/media processing for Bedrock conversations.""" + + @staticmethod + def _extract_video_url(element: Union[dict, "ChatCompletionVideoObject"]) -> str: + """Extract the URL string from a video_url content element.""" + if isinstance(element["video_url"], dict): + return element["video_url"]["url"] + return element["video_url"] @staticmethod def _post_call_image_processing( @@ -3717,20 +3728,98 @@ class BedrockImageProcessor: image=BedrockImageBlock(source=_blob, format=image_format) ) + @staticmethod + def _get_bedrock_format_from_extension(extension: str) -> str: + """Map file extension to Bedrock format enum value.""" + # 3gpp is an alias for 3gp + if extension == "3gpp": + return "3gp" + # jpg → jpeg + if extension == "jpg": + return "jpeg" + return extension + + @classmethod + def _create_s3_bedrock_block( + cls, s3_url: str, format: Optional[str] = None + ) -> BedrockContentBlock: + """ + Create a Bedrock content block referencing an S3 object. + Determines block type (image/document/video) from file extension, + or from the explicit ``format`` override if provided. + """ + # If explicit format override provided (e.g. from file.format field), + # use it directly instead of requiring a file extension. + if format: + raw_format = format.split("/")[-1] if "/" in format else format + bedrock_format = cls._get_bedrock_format_from_extension(raw_format.lower()) + else: + extension = os.path.splitext(s3_url)[-1].lstrip(".").lower() + if not extension: + raise ValueError( + f"Cannot determine file type from S3 URL (no extension): {s3_url}" + ) + bedrock_format = cls._get_bedrock_format_from_extension(extension) + + s3_source = BedrockSourceBlock(s3Location=BedrockS3Location(uri=s3_url)) + + config = litellm.AmazonConverseConfig() + supported_image_formats = config.get_supported_image_types() + supported_doc_formats = config.get_supported_document_types() + supported_video_formats = config.get_supported_video_types() + + if bedrock_format in supported_image_formats: + return BedrockContentBlock( + image=BedrockImageBlock(source=s3_source, format=bedrock_format) + ) + elif bedrock_format in supported_doc_formats: + doc_name = f"s3doc_{hashlib.sha256(s3_url.encode()).hexdigest()[:16]}_{bedrock_format}" + return BedrockContentBlock( + document=BedrockDocumentBlock( + source=s3_source, format=bedrock_format, name=doc_name + ) + ) + elif bedrock_format in supported_video_formats: + return BedrockContentBlock( + video=BedrockVideoBlock(source=s3_source, format=bedrock_format) + ) + else: + raise ValueError( + f"Unsupported file format '{bedrock_format}' for Bedrock S3 content. " + f"Supported: images={supported_image_formats}, " + f"documents={supported_doc_formats}, " + f"videos={supported_video_formats}" + ) + + # TODO: Rename to process_media_sync/async — these methods now handle images, documents, and videos (not just images). @classmethod def process_image_sync( - cls, image_url: str, format: Optional[str] = None + cls, + image_url: str, + format: Optional[str] = None, + model: Optional[str] = None, + custom_llm_provider: Optional[str] = None, ) -> BedrockContentBlock: - """Synchronous image processing.""" + """Synchronous processing of media URLs (images, documents, videos) for Bedrock.""" - if "base64" in image_url: + if image_url.startswith("s3://"): + if model and not supports_s3_input( + model=model, custom_llm_provider=custom_llm_provider + ): + raise ValueError( + f"Model '{model}' does not support s3:// URLs. " + "Only Amazon Nova models (with vision) support S3 input via Bedrock Converse API. " + "Please use a base64-encoded or https:// URL instead." + ) + return cls._create_s3_bedrock_block(image_url, format) + elif "base64" in image_url: img_bytes, mime_type, image_format = cls._parse_base64_image(image_url) elif "http://" in image_url or "https://" in image_url: img_bytes, mime_type = BedrockImageProcessor.get_image_details(image_url) image_format = mime_type.split("/")[1] else: raise ValueError( - "Unsupported image type. Expected either image url or base64 encoded string" + "Unsupported image type. Expected either image url, base64 encoded string, or s3:// URL" ) if format: @@ -3742,11 +3831,25 @@ class BedrockImageProcessor: @classmethod async def process_image_async( - cls, image_url: str, format: Optional[str] + cls, + image_url: str, + format: Optional[str], + model: Optional[str] = None, + custom_llm_provider: Optional[str] = None, ) -> BedrockContentBlock: - """Asynchronous image processing.""" + """Asynchronous processing of media URLs (images, documents, videos) for Bedrock.""" - if "base64" in image_url: + if image_url.startswith("s3://"): + if model and not supports_s3_input( + model=model, custom_llm_provider=custom_llm_provider + ): + raise ValueError( + f"Model '{model}' does not support s3:// URLs. " + "Only Amazon Nova models (with vision) support S3 input via Bedrock Converse API. " + "Please use a base64-encoded or https:// URL instead." + ) + return cls._create_s3_bedrock_block(image_url, format) + elif "base64" in image_url: img_bytes, mime_type, image_format = cls._parse_base64_image(image_url) elif "http://" in image_url or "https://" in image_url: img_bytes, mime_type = await BedrockImageProcessor.get_image_details_async( @@ -3755,7 +3858,7 @@ class BedrockImageProcessor: image_format = mime_type.split("/")[1] else: raise ValueError( - "Unsupported image type. Expected either image url or base64 encoded string" + "Unsupported image type. Expected either image url, base64 encoded string, or s3:// URL" ) if format: # override with user-defined params @@ -4396,14 +4499,30 @@ class BedrockConverseMessagesProcessor: else: image_url = element["image_url"] _part = await BedrockImageProcessor.process_image_async( # type: ignore - image_url=image_url, format=format + image_url=image_url, + format=format, + model=model, + custom_llm_provider=llm_provider, ) _parts.append(_part) # type: ignore elif element["type"] == "file": _part = await BedrockConverseMessagesProcessor._async_process_file_message( - message=cast(ChatCompletionFileObject, element) + message=cast(ChatCompletionFileObject, element), + model=model, + llm_provider=llm_provider, ) _parts.append(_part) + elif element["type"] == "video_url": + video_url = BedrockImageProcessor._extract_video_url( + element + ) + _part = await BedrockImageProcessor.process_image_async( # type: ignore + image_url=video_url, + format=None, + model=model, + custom_llm_provider=llm_provider, + ) + _parts.append(_part) # type: ignore _cache_point_block = ( litellm.AmazonConverseConfig()._get_cache_point_block( message_block=cast( @@ -4645,7 +4764,11 @@ class BedrockConverseMessagesProcessor: return reasoning_content_blocks @staticmethod - def _process_file_message(message: ChatCompletionFileObject) -> BedrockContentBlock: + def _process_file_message( + message: ChatCompletionFileObject, + model: Optional[str] = None, + llm_provider: Optional[str] = None, + ) -> BedrockContentBlock: file_message = message["file"] file_data = file_message.get("file_data") file_id = file_message.get("file_id") @@ -4660,12 +4783,17 @@ class BedrockConverseMessagesProcessor: ) format = file_message.get("format") return BedrockImageProcessor.process_image_sync( - image_url=cast(str, file_id or file_data), format=format + image_url=cast(str, file_id or file_data), + format=format, + model=model, + custom_llm_provider=llm_provider, ) @staticmethod async def _async_process_file_message( message: ChatCompletionFileObject, + model: Optional[str] = None, + llm_provider: Optional[str] = None, ) -> BedrockContentBlock: file_message = message["file"] file_data = file_message.get("file_data") @@ -4680,7 +4808,10 @@ class BedrockConverseMessagesProcessor: llm_provider="bedrock", ) return await BedrockImageProcessor.process_image_async( - image_url=cast(str, file_id or file_data), format=format + image_url=cast(str, file_id or file_data), + format=format, + model=model, + custom_llm_provider=llm_provider, ) @staticmethod @@ -4771,15 +4902,30 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915 _part = BedrockImageProcessor.process_image_sync( # type: ignore image_url=image_url, format=format, + model=model, + custom_llm_provider=llm_provider, ) _parts.append(_part) # type: ignore elif element["type"] == "file": _part = ( BedrockConverseMessagesProcessor._process_file_message( - message=cast(ChatCompletionFileObject, element) + message=cast(ChatCompletionFileObject, element), + model=model, + llm_provider=llm_provider, ) ) _parts.append(_part) + elif element["type"] == "video_url": + video_url = BedrockImageProcessor._extract_video_url( + element + ) + _part = BedrockImageProcessor.process_image_sync( # type: ignore + image_url=video_url, + format=None, + model=model, + custom_llm_provider=llm_provider, + ) + _parts.append(_part) # type: ignore _cache_point_block = ( litellm.AmazonConverseConfig()._get_cache_point_block( message_block=cast( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 879dd42be47..ac6bc568dea 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -335,7 +335,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -352,7 +353,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 5.46875e-07, @@ -371,7 +373,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "apac.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 8.25e-08, @@ -388,7 +391,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "apac.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 5.46875e-07, @@ -407,7 +411,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "eu.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 8.25e-08, @@ -424,7 +429,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "eu.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 5.46875e-07, @@ -443,7 +449,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "us.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 8.25e-08, @@ -460,7 +467,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "us.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 5.46875e-07, @@ -479,7 +487,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", @@ -522,7 +531,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "amazon.rerank-v1:0": { "input_cost_per_query": 0.001, @@ -1478,7 +1488,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "apac.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.7e-08, @@ -1504,7 +1515,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -7582,7 +7594,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "bedrock/us-gov-east-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -7713,7 +7726,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "bedrock/us-gov-west-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -12115,7 +12129,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "eu.amazon.nova-micro-v1:0": { "input_cost_per_token": 4.6e-08, @@ -12142,7 +12157,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { "input_cost_per_token": 2.5e-07, @@ -14599,17 +14615,14 @@ "uses_embed_content": true }, "vertex_ai/gemini-embedding-2-preview": { - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, - "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_token": 1.5e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal", "supports_multimodal": true, "uses_embed_content": true }, @@ -14626,18 +14639,6 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "uses_embed_content": true }, - "vertex_ai/gemini-embedding-2-preview": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "vertex_ai", - "max_input_tokens": 8192, - "max_tokens": 8192, - "mode": "embedding", - "output_cost_per_token": 0, - "output_vector_size": 3072, - "source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal", - "supports_multimodal": true, - "uses_embed_content": true - }, "gemini/gemini-embedding-001": { "input_cost_per_token": 1.5e-07, "litellm_provider": "gemini", @@ -16834,7 +16835,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "gpt-3.5-turbo": { "input_cost_per_token": 5e-07, @@ -28050,7 +28052,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "us.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.5e-08, @@ -28076,7 +28079,8 @@ "supports_pdf_input": true, "supports_prompt_caching": false, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "us.amazon.nova-pro-v1:0": { "input_cost_per_token": 8e-07, @@ -28090,7 +28094,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -31013,7 +31018,9 @@ "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index 54237dfb37a..387a04e4a77 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -16,8 +16,14 @@ class SystemContentBlock(TypedDict, total=False): cachePoint: CachePointBlock -class SourceBlock(TypedDict): +class S3Location(TypedDict, total=False): + uri: Required[str] # s3://bucket/key + + +class SourceBlock(TypedDict, total=False): + # Mutually exclusive: provide either `bytes` (inline base64) or `s3Location` (S3 reference), not both. bytes: Optional[str] # base 64 encoded string + s3Location: S3Location # S3 reference for large content BedrockImageTypes = Literal["png", "jpeg", "gif", "webp"] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 38425c7ac4a..a27f95735cd 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -131,6 +131,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_embedding_image_input: Optional[bool] supports_audio_output: Optional[bool] supports_pdf_input: Optional[bool] + supports_s3_input: Optional[bool] supports_native_streaming: Optional[bool] supports_parallel_function_calling: Optional[bool] supports_web_search: Optional[bool] diff --git a/litellm/utils.py b/litellm/utils.py index c8272586dad..e4bd5e2a826 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2590,6 +2590,13 @@ def supports_pdf_input(model: str, custom_llm_provider: Optional[str] = None) -> ) +def supports_s3_input(model: str, custom_llm_provider: Optional[str] = None) -> bool: + """Check if a given model supports s3:// URLs as content source in Bedrock Converse API""" + return _supports_factory( + model=model, custom_llm_provider=custom_llm_provider, key="supports_s3_input" + ) + + def supports_audio_output( model: str, custom_llm_provider: Optional[str] = None ) -> bool: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 879dd42be47..ac6bc568dea 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -335,7 +335,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -352,7 +353,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 5.46875e-07, @@ -371,7 +373,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "apac.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 8.25e-08, @@ -388,7 +391,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "apac.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 5.46875e-07, @@ -407,7 +411,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "eu.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 8.25e-08, @@ -424,7 +429,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "eu.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 5.46875e-07, @@ -443,7 +449,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "us.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 8.25e-08, @@ -460,7 +467,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "us.amazon.nova-2-pro-preview-20251202-v1:0": { "cache_read_input_token_cost": 5.46875e-07, @@ -479,7 +487,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "amazon.nova-2-multimodal-embeddings-v1:0": { "litellm_provider": "bedrock", @@ -522,7 +531,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "amazon.rerank-v1:0": { "input_cost_per_query": 0.001, @@ -1478,7 +1488,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "apac.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.7e-08, @@ -1504,7 +1515,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -7582,7 +7594,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "bedrock/us-gov-east-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -7713,7 +7726,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "bedrock/us-gov-west-1/amazon.titan-embed-text-v1": { "input_cost_per_token": 1e-07, @@ -12115,7 +12129,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "eu.amazon.nova-micro-v1:0": { "input_cost_per_token": 4.6e-08, @@ -12142,7 +12157,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { "input_cost_per_token": 2.5e-07, @@ -14599,17 +14615,14 @@ "uses_embed_content": true }, "vertex_ai/gemini-embedding-2-preview": { - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, - "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_token": 1.5e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0, "output_vector_size": 3072, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal", "supports_multimodal": true, "uses_embed_content": true }, @@ -14626,18 +14639,6 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", "uses_embed_content": true }, - "vertex_ai/gemini-embedding-2-preview": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "vertex_ai", - "max_input_tokens": 8192, - "max_tokens": 8192, - "mode": "embedding", - "output_cost_per_token": 0, - "output_vector_size": 3072, - "source": "https://ai.google.dev/gemini-api/docs/embeddings#multimodal", - "supports_multimodal": true, - "uses_embed_content": true - }, "gemini/gemini-embedding-001": { "input_cost_per_token": 1.5e-07, "litellm_provider": "gemini", @@ -16834,7 +16835,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "gpt-3.5-turbo": { "input_cost_per_token": 5e-07, @@ -28050,7 +28052,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "us.amazon.nova-micro-v1:0": { "input_cost_per_token": 3.5e-08, @@ -28076,7 +28079,8 @@ "supports_pdf_input": true, "supports_prompt_caching": false, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "us.amazon.nova-pro-v1:0": { "input_cost_per_token": 8e-07, @@ -28090,7 +28094,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_vision": true + "supports_vision": true, + "supports_s3_input": true }, "us.anthropic.claude-3-5-haiku-20241022-v1:0": { "cache_creation_input_token_cost": 1e-06, @@ -31013,7 +31018,9 @@ "mode": "chat", "output_cost_per_token": 3.2e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#glm-models", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, diff --git a/tests/local_testing/test_bedrock_s3_content.py b/tests/local_testing/test_bedrock_s3_content.py new file mode 100644 index 00000000000..b6bc5ddfc37 --- /dev/null +++ b/tests/local_testing/test_bedrock_s3_content.py @@ -0,0 +1,183 @@ +""" +Integration tests for Bedrock S3 URL pass-through support. + +These tests make real requests to AWS Bedrock and S3. +Requires AWS credentials and a pre-populated S3 bucket. + +To run: + AWS_PROFILE= BEDROCK_S3_TEST_BUCKET= \ + pytest tests/local_testing/test_bedrock_s3_content.py -v + +Required env vars (one of): + AWS_PROFILE — named profile with Bedrock + S3 access + AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY + +Required: + BEDROCK_S3_TEST_BUCKET — S3 bucket containing test files + +Expected test files in the bucket: + photos/cat.jpg, docs/report.pdf, videos/demo.mp4 +""" + +import os + +import pytest + +_has_aws_creds = bool( + os.environ.get("AWS_PROFILE") + or (os.environ.get("AWS_ACCESS_KEY_ID") and os.environ.get("AWS_SECRET_ACCESS_KEY")) +) +_s3_bucket = os.environ.get("BEDROCK_S3_TEST_BUCKET", "") +_skip_reason = "Requires AWS credentials (AWS_PROFILE or AWS_ACCESS_KEY_ID/SECRET) and BEDROCK_S3_TEST_BUCKET" + + +@pytest.fixture(autouse=True) +def _ensure_s3_input_flag(monkeypatch): + """Patch supports_s3_input in factory.py for Nova models. + + Until this PR merges upstream, the remote model_prices JSON (fetched at + import time) won't contain the supports_s3_input flag. We monkeypatch + the function directly in factory.py where it's imported. + """ + + def _patched(model, custom_llm_provider=None): + model_lower = model.lower() + return ( + "nova" in model_lower + and "micro" not in model_lower + and "embed" not in model_lower + ) + + monkeypatch.setattr( + "litellm.litellm_core_utils.prompt_templates.factory.supports_s3_input", + _patched, + ) + + +@pytest.mark.skipif(not (_has_aws_creds and _s3_bucket), reason=_skip_reason) +class TestS3IntegrationBedrock: + """Integration tests that send real requests to Bedrock with S3 content.""" + + _MODELS = [ + "bedrock/us.amazon.nova-lite-v1:0", + "bedrock/us.amazon.nova-pro-v1:0", + "bedrock/us.amazon.nova-2-lite-v1:0", + ] + + @pytest.mark.parametrize("model", _MODELS) + def test_should_send_s3_image(self, model): + import litellm + + response = litellm.completion( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image in one sentence."}, + { + "type": "image_url", + "image_url": {"url": f"s3://{_s3_bucket}/photos/cat.jpg"}, + }, + ], + } + ], + max_tokens=100, + ) + assert response.choices[0].message.content is not None + assert len(response.choices[0].message.content) > 0 + + @pytest.mark.parametrize("model", _MODELS) + def test_should_send_s3_pdf(self, model): + import litellm + + response = litellm.completion( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this document in one sentence."}, + { + "type": "file", + "file": {"file_data": f"s3://{_s3_bucket}/docs/report.pdf"}, + }, + ], + } + ], + max_tokens=100, + ) + assert response.choices[0].message.content is not None + assert len(response.choices[0].message.content) > 0 + + @pytest.mark.parametrize("model", _MODELS) + def test_should_send_s3_video(self, model): + import litellm + + response = litellm.completion( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video in one sentence."}, + { + "type": "video_url", + "video_url": {"url": f"s3://{_s3_bucket}/videos/demo.mp4"}, + }, + ], + } + ], + max_tokens=100, + ) + assert response.choices[0].message.content is not None + assert len(response.choices[0].message.content) > 0 + + @pytest.mark.parametrize("model", _MODELS) + def test_should_send_multiple_s3_files_in_one_message(self, model): + import litellm + + response = litellm.completion( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe what you see in these two items."}, + { + "type": "image_url", + "image_url": {"url": f"s3://{_s3_bucket}/photos/cat.jpg"}, + }, + { + "type": "file", + "file": {"file_data": f"s3://{_s3_bucket}/docs/report.pdf"}, + }, + ], + } + ], + max_tokens=200, + ) + assert response.choices[0].message.content is not None + assert len(response.choices[0].message.content) > 0 + + def test_should_reject_s3_url_for_unsupported_model(self): + """Claude models don't support s3Location — should get a clear error.""" + import litellm + + with pytest.raises(Exception, match="does not support s3://"): + litellm.completion( + model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + { + "type": "image_url", + "image_url": {"url": f"s3://{_s3_bucket}/photos/cat.jpg"}, + }, + ], + } + ], + max_tokens=50, + ) diff --git a/tests/test_litellm/llms/bedrock/chat/test_s3_content_blocks.py b/tests/test_litellm/llms/bedrock/chat/test_s3_content_blocks.py new file mode 100644 index 00000000000..956f71ede63 --- /dev/null +++ b/tests/test_litellm/llms/bedrock/chat/test_s3_content_blocks.py @@ -0,0 +1,468 @@ +""" +Tests for Bedrock S3 URL pass-through support (PR1). + +When users provide s3:// URLs in OpenAI-format messages, LiteLLM should +map them to Bedrock's native s3Location blocks without downloading or +base64-encoding. +""" + +import hashlib +import os +from unittest.mock import patch + +import pytest + +from litellm.litellm_core_utils.prompt_templates.factory import BedrockImageProcessor + + +def _mock_supports_s3(model: str, custom_llm_provider=None) -> bool: + """Mock supports_s3_input that returns True for Nova models.""" + return "nova" in model.lower() and "micro" not in model.lower() + + +class TestS3ImageBlocks: + """Test S3 URL → ImageBlock creation.""" + + def test_should_create_s3_image_block_from_jpg_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/image.jpg") + assert result["image"]["format"] == "jpeg" + assert result["image"]["source"]["s3Location"]["uri"] == "s3://bucket/image.jpg" + assert "bytes" not in result["image"]["source"] + + def test_should_create_s3_image_block_from_png_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/image.png") + assert result["image"]["format"] == "png" + assert result["image"]["source"]["s3Location"]["uri"] == "s3://bucket/image.png" + + def test_should_create_s3_image_block_from_gif_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/image.gif") + assert result["image"]["format"] == "gif" + + def test_should_create_s3_image_block_from_webp_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/image.webp") + assert result["image"]["format"] == "webp" + + def test_should_create_s3_image_block_from_jpeg_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/photo.jpeg") + assert result["image"]["format"] == "jpeg" + + +class TestS3DocumentBlocks: + """Test S3 URL → DocumentBlock creation.""" + + def test_should_create_s3_document_block_from_pdf_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/doc.pdf") + assert result["document"]["format"] == "pdf" + assert ( + result["document"]["source"]["s3Location"]["uri"] == "s3://bucket/doc.pdf" + ) + assert "name" in result["document"] + assert "bytes" not in result["document"]["source"] + + def test_should_create_s3_document_block_from_docx_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/doc.docx") + assert result["document"]["format"] == "docx" + + def test_should_create_s3_document_block_from_xlsx_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/data.xlsx") + assert result["document"]["format"] == "xlsx" + + def test_should_create_s3_document_block_from_md_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/readme.md") + assert result["document"]["format"] == "md" + + def test_should_create_s3_document_block_from_txt_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/notes.txt") + assert result["document"]["format"] == "txt" + + def test_should_create_s3_document_block_from_csv_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/data.csv") + assert result["document"]["format"] == "csv" + + def test_should_create_s3_document_block_from_html_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/page.html") + assert result["document"]["format"] == "html" + + def test_should_generate_deterministic_document_names(self): + r1 = BedrockImageProcessor.process_image_sync("s3://bucket/doc.pdf") + r2 = BedrockImageProcessor.process_image_sync("s3://bucket/doc.pdf") + assert r1["document"]["name"] == r2["document"]["name"] + + r3 = BedrockImageProcessor.process_image_sync("s3://bucket/other.pdf") + assert r1["document"]["name"] != r3["document"]["name"] + + +class TestS3VideoBlocks: + """Test S3 URL → VideoBlock creation.""" + + def test_should_create_s3_video_block_from_mp4_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/video.mp4") + assert result["video"]["format"] == "mp4" + assert result["video"]["source"]["s3Location"]["uri"] == "s3://bucket/video.mp4" + assert "bytes" not in result["video"]["source"] + + def test_should_create_s3_video_block_from_mov_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/video.mov") + assert result["video"]["format"] == "mov" + + def test_should_create_s3_video_block_from_mkv_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/video.mkv") + assert result["video"]["format"] == "mkv" + + def test_should_create_s3_video_block_from_3gp_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/video.3gp") + assert result["video"]["format"] == "3gp" + + def test_should_create_s3_video_block_from_3gpp_url(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/video.3gpp") + assert result["video"]["format"] == "3gp" + + +class TestS3ErrorCases: + """Test error handling for S3 URLs.""" + + def test_should_raise_error_for_s3_url_without_extension(self): + with pytest.raises(ValueError, match="no extension"): + BedrockImageProcessor.process_image_sync("s3://bucket/noext") + + def test_should_raise_error_for_unsupported_extension(self): + with pytest.raises(ValueError, match="Unsupported file format"): + BedrockImageProcessor.process_image_sync("s3://bucket/file.xyz") + + def test_should_not_raise_for_s3_url_without_extension_when_format_provided(self): + """format override bypasses extension requirement.""" + result = BedrockImageProcessor.process_image_sync( + "s3://bucket/blob", format="image/png" + ) + assert result["image"]["format"] == "png" + + +class TestS3EdgeCases: + """Test edge cases: uppercase extensions, dots in path, etc.""" + + def test_should_handle_uppercase_extension(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/IMAGE.JPG") + assert result["image"]["format"] == "jpeg" + + def test_should_handle_mixed_case_extension(self): + result = BedrockImageProcessor.process_image_sync("s3://bucket/doc.Pdf") + assert result["document"]["format"] == "pdf" + + def test_should_handle_dots_in_path(self): + result = BedrockImageProcessor.process_image_sync( + "s3://bucket/path/to/file.with.dots.jpg" + ) + assert result["image"]["format"] == "jpeg" + assert ( + result["image"]["source"]["s3Location"]["uri"] + == "s3://bucket/path/to/file.with.dots.jpg" + ) + + def test_should_handle_deeply_nested_s3_path(self): + url = "s3://my-bucket/a/b/c/d/report.pdf" + result = BedrockImageProcessor.process_image_sync(url) + assert result["document"]["format"] == "pdf" + assert result["document"]["source"]["s3Location"]["uri"] == url + + +class TestS3FormatOverride: + """Test explicit format override for S3 URLs.""" + + def test_should_handle_s3_url_with_explicit_format_override(self): + # Even though extension is .dat, explicit format says pdf + result = BedrockImageProcessor.process_image_sync( + "s3://bucket/file.dat", format="application/pdf" + ) + assert result["document"]["format"] == "pdf" + + def test_should_handle_s3_url_with_simple_format_override(self): + result = BedrockImageProcessor.process_image_sync( + "s3://bucket/file.dat", format="png" + ) + assert result["image"]["format"] == "png" + + +class TestS3AsyncProcessing: + """Test async S3 URL processing.""" + + @pytest.mark.asyncio + async def test_should_create_s3_image_block_async(self): + result = await BedrockImageProcessor.process_image_async( + "s3://bucket/image.jpg", format=None + ) + assert result["image"]["format"] == "jpeg" + assert result["image"]["source"]["s3Location"]["uri"] == "s3://bucket/image.jpg" + + @pytest.mark.asyncio + async def test_should_create_s3_document_block_async(self): + result = await BedrockImageProcessor.process_image_async( + "s3://bucket/doc.pdf", format=None + ) + assert result["document"]["format"] == "pdf" + + @pytest.mark.asyncio + async def test_should_create_s3_video_block_async(self): + result = await BedrockImageProcessor.process_image_async( + "s3://bucket/video.mp4", format=None + ) + assert result["video"]["format"] == "mp4" + + +class TestExistingPathsUnbroken: + """Ensure existing base64 and HTTPS paths still work.""" + + def test_should_still_handle_base64_urls(self): + # Minimal valid base64 PNG + b64_url = "data:image/png;base64,iVBORw0KGgo=" + result = BedrockImageProcessor.process_image_sync(b64_url) + assert result["image"]["format"] == "png" + assert "bytes" in result["image"]["source"] + assert "s3Location" not in result["image"]["source"] + + +@patch( + "litellm.litellm_core_utils.prompt_templates.factory.supports_s3_input", + _mock_supports_s3, +) +class TestFullMessageTransformation: + """Test S3 URLs through full Bedrock message transformation.""" + + def test_should_pass_s3_image_url_through_full_message_transformation(self): + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + { + "type": "image_url", + "image_url": {"url": "s3://my-bucket/photos/cat.jpg"}, + }, + ], + } + ] + result = _bedrock_converse_messages_pt( + messages=messages, + model="us.amazon.nova-lite-v1:0", + llm_provider="bedrock", + ) + # Find the image block in the result + content_blocks = result[0]["content"] + image_blocks = [b for b in content_blocks if "image" in b] + assert len(image_blocks) == 1 + assert ( + image_blocks[0]["image"]["source"]["s3Location"]["uri"] + == "s3://my-bucket/photos/cat.jpg" + ) + assert image_blocks[0]["image"]["format"] == "jpeg" + + def test_should_pass_s3_video_url_through_full_message_transformation(self): + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video"}, + { + "type": "video_url", + "video_url": {"url": "s3://my-bucket/videos/demo.mp4"}, + }, + ], + } + ] + result = _bedrock_converse_messages_pt( + messages=messages, + model="us.amazon.nova-pro-v1:0", + llm_provider="bedrock", + ) + content_blocks = result[0]["content"] + video_blocks = [b for b in content_blocks if "video" in b] + assert len(video_blocks) == 1 + assert ( + video_blocks[0]["video"]["source"]["s3Location"]["uri"] + == "s3://my-bucket/videos/demo.mp4" + ) + assert video_blocks[0]["video"]["format"] == "mp4" + + def test_should_pass_s3_file_url_through_full_message_transformation(self): + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Summarize this document"}, + { + "type": "file", + "file": { + "file_data": "s3://my-bucket/docs/report.pdf", + }, + }, + ], + } + ] + result = _bedrock_converse_messages_pt( + messages=messages, + model="us.amazon.nova-lite-v1:0", + llm_provider="bedrock", + ) + content_blocks = result[0]["content"] + doc_blocks = [b for b in content_blocks if "document" in b] + assert len(doc_blocks) == 1 + assert ( + doc_blocks[0]["document"]["source"]["s3Location"]["uri"] + == "s3://my-bucket/docs/report.pdf" + ) + assert doc_blocks[0]["document"]["format"] == "pdf" + + @pytest.mark.asyncio + async def test_should_pass_s3_video_url_through_async_message_transformation(self): + from litellm.litellm_core_utils.prompt_templates.factory import ( + BedrockConverseMessagesProcessor, + ) + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this video"}, + { + "type": "video_url", + "video_url": {"url": "s3://my-bucket/videos/demo.mp4"}, + }, + ], + } + ] + result = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, + model="us.amazon.nova-pro-v1:0", + llm_provider="bedrock", + ) + ) + content_blocks = result[0]["content"] + video_blocks = [b for b in content_blocks if "video" in b] + assert len(video_blocks) == 1 + assert ( + video_blocks[0]["video"]["source"]["s3Location"]["uri"] + == "s3://my-bucket/videos/demo.mp4" + ) + assert video_blocks[0]["video"]["format"] == "mp4" + + def test_should_handle_video_url_as_plain_string(self): + """video_url can be a plain string instead of a dict.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe"}, + {"type": "video_url", "video_url": "s3://bucket/video.mp4"}, + ], + } + ] + result = _bedrock_converse_messages_pt( + messages=messages, + model="us.amazon.nova-pro-v1:0", + llm_provider="bedrock", + ) + content_blocks = result[0]["content"] + video_blocks = [b for b in content_blocks if "video" in b] + assert len(video_blocks) == 1 + assert video_blocks[0]["video"]["format"] == "mp4" + + def test_should_still_handle_https_image_url(self): + """Regression: HTTPS URLs should still be downloaded and base64-encoded, not treated as S3.""" + from unittest.mock import patch + + from litellm.litellm_core_utils.prompt_templates.factory import ( + _bedrock_converse_messages_pt, + ) + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this."}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/photo.jpg"}, + }, + ], + } + ] + + fake_bytes = b"\x89PNG\r\n\x1a\n" + b"\x00" * 100 + with patch.object( + BedrockImageProcessor, + "get_image_details", + return_value=(fake_bytes, "image/jpeg"), + ): + result = _bedrock_converse_messages_pt( + messages=messages, + model="us.amazon.nova-lite-v1:0", + llm_provider="bedrock", + ) + content_blocks = result[0]["content"] + img_blocks = [b for b in content_blocks if "image" in b] + assert len(img_blocks) == 1 + # Should have inline bytes, NOT s3Location + assert "bytes" in img_blocks[0]["image"]["source"] + assert "s3Location" not in img_blocks[0]["image"]["source"] + + def test_should_accept_s3_video_url_in_extract(self): + """s3:// video URLs should pass through extraction.""" + url = BedrockImageProcessor._extract_video_url( + {"type": "video_url", "video_url": {"url": "s3://bucket/video.mp4"}} + ) + assert url == "s3://bucket/video.mp4" + + +@patch( + "litellm.litellm_core_utils.prompt_templates.factory.supports_s3_input", + _mock_supports_s3, +) +class TestS3ModelGuard: + """Test that S3 URLs are rejected for models that don't support s3Location.""" + + def test_should_reject_s3_url_for_unsupported_model(self): + with pytest.raises(ValueError, match="does not support s3://"): + BedrockImageProcessor.process_image_sync( + "s3://bucket/image.jpg", + model="anthropic.claude-3-sonnet-20240229-v1:0", + custom_llm_provider="bedrock", + ) + + def test_should_allow_s3_url_for_nova_model(self): + result = BedrockImageProcessor.process_image_sync( + "s3://bucket/image.jpg", + model="us.amazon.nova-lite-v1:0", + custom_llm_provider="bedrock", + ) + assert result["image"]["source"]["s3Location"]["uri"] == "s3://bucket/image.jpg" + + def test_should_allow_s3_url_when_model_not_specified(self): + """When no model is provided, allow S3 pass-through (caller's responsibility).""" + result = BedrockImageProcessor.process_image_sync("s3://bucket/image.jpg") + assert result["image"]["source"]["s3Location"]["uri"] == "s3://bucket/image.jpg" + + @pytest.mark.asyncio + async def test_should_reject_s3_url_for_unsupported_model_async(self): + with pytest.raises(ValueError, match="does not support s3://"): + await BedrockImageProcessor.process_image_async( + "s3://bucket/image.jpg", + format=None, + model="anthropic.claude-3-sonnet-20240229-v1:0", + custom_llm_provider="bedrock", + ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 64488e2fb6a..0e51a5a9b2c 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -734,6 +734,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_pdf_input": {"type": "boolean"}, "supports_prompt_caching": {"type": "boolean"}, "supports_response_schema": {"type": "boolean"}, + "supports_s3_input": {"type": "boolean"}, "supports_system_messages": {"type": "boolean"}, "supports_tool_choice": {"type": "boolean"}, "supports_video_input": {"type": "boolean"},