diff --git a/docs/my-website/docs/providers/bedrock_embedding.md b/docs/my-website/docs/providers/bedrock_embedding.md index e2e7c0dcedd..3c618fe0641 100644 --- a/docs/my-website/docs/providers/bedrock_embedding.md +++ b/docs/my-website/docs/providers/bedrock_embedding.md @@ -172,6 +172,125 @@ print(f"Results available at: {output_s3_uri}") **Note:** The actual embedding results are stored in S3. When the job is completed, download the results from the S3 location specified in `status.metadata['output_file_id']`. The results will be in JSON/JSONL format containing the embedding vectors. +## Amazon Nova Multimodal Embeddings + +Amazon Nova supports multimodal embeddings for text, images, video, and audio. It offers flexible embedding dimensions and purposes optimized for different use cases. + +### Supported Features + +- **Modalities**: Text, Image, Video, Audio +- **Dimensions**: 256, 384, 1024, 3072 (default: 3072) +- **Embedding Purposes**: + - `GENERIC_INDEX` (default) + - `GENERIC_RETRIEVAL` + - `TEXT_RETRIEVAL` + - `IMAGE_RETRIEVAL` + - `VIDEO_RETRIEVAL` + - `AUDIO_RETRIEVAL` + - `CLASSIFICATION` + - `CLUSTERING` + +### Text Embedding + +```python +from litellm import embedding + +response = embedding( + model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", + input=["Hello, world!"], + aws_region_name="us-east-1", + dimensions=1024, # Optional: 256, 384, 1024, or 3072 +) + +print(response.data[0].embedding) +``` + +### Image Embedding with Base64 + +Amazon Nova accepts images in base64 format using the standard data URL format: + +```python +import base64 +from litellm import embedding + +# Method 1: Load image from file +with open("image.jpg", "rb") as image_file: + image_data = base64.b64encode(image_file.read()).decode('utf-8') + # Create data URL with proper format + image_base64 = f"data:image/jpeg;base64,{image_data}" + +response = embedding( + model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", + input=[image_base64], + aws_region_name="us-east-1", + dimensions=1024, +) + +print(f"Image embedding: {response.data[0].embedding[:10]}...") # First 10 dimensions +``` + +#### Supported Image Formats + +Nova supports the following image formats: +- JPEG: `data:image/jpeg;base64,...` +- PNG: `data:image/png;base64,...` +- GIF: `data:image/gif;base64,...` +- WebP: `data:image/webp;base64,...` + +#### Complete Example with Error Handling + +```python +import base64 +from litellm import embedding + +def get_image_embedding(image_path, dimensions=1024): + """ + Get embedding for an image file. + + Args: + image_path: Path to the image file + dimensions: Embedding dimension (256, 384, 1024, or 3072) + + Returns: + List of embedding values + """ + try: + # Determine image format from file extension + if image_path.lower().endswith('.png'): + mime_type = "image/png" + elif image_path.lower().endswith(('.jpg', '.jpeg')): + mime_type = "image/jpeg" + elif image_path.lower().endswith('.gif'): + mime_type = "image/gif" + elif image_path.lower().endswith('.webp'): + mime_type = "image/webp" + else: + raise ValueError(f"Unsupported image format: {image_path}") + + # Read and encode image + with open(image_path, "rb") as image_file: + image_data = base64.b64encode(image_file.read()).decode('utf-8') + image_base64 = f"data:{mime_type};base64,{image_data}" + + # Get embedding + response = embedding( + model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", + input=[image_base64], + aws_region_name="us-east-1", + dimensions=dimensions, + ) + + return response.data[0].embedding + + except Exception as e: + print(f"Error getting image embedding: {e}") + raise + +# Example usage +image_embedding = get_image_embedding("photo.jpg", dimensions=1024) +print(f"Got embedding with {len(image_embedding)} dimensions") +``` + ### Error Handling #### Common Errors diff --git a/litellm/llms/bedrock/embed/amazon_nova_transformation.py b/litellm/llms/bedrock/embed/amazon_nova_transformation.py index ada49d0ff21..3e5686c46fb 100644 --- a/litellm/llms/bedrock/embed/amazon_nova_transformation.py +++ b/litellm/llms/bedrock/embed/amazon_nova_transformation.py @@ -46,6 +46,39 @@ class AmazonNovaEmbeddingConfig: elif k in self.get_supported_openai_params(): optional_params[k] = v return optional_params + + def _parse_data_url(self, data_url: str) -> tuple: + """ + Parse a data URL to extract the media type and base64 data. + + Args: + data_url: Data URL in format: data:image/jpeg;base64,/9j/4AAQ... + + Returns: + tuple: (media_type, base64_data) + media_type: e.g., "image/jpeg", "video/mp4", "audio/mpeg" + base64_data: The base64-encoded data without the prefix + """ + if not data_url.startswith("data:"): + raise ValueError(f"Invalid data URL format: {data_url[:50]}...") + + # Split by comma to separate metadata from data + # Format: data:image/jpeg;base64, + if "," not in data_url: + raise ValueError(f"Invalid data URL format (missing comma): {data_url[:50]}...") + + metadata, base64_data = data_url.split(",", 1) + + # Extract media type from metadata + # Remove 'data:' prefix and ';base64' suffix + metadata = metadata[5:] # Remove 'data:' + + if ";" in metadata: + media_type = metadata.split(";")[0] + else: + media_type = metadata + + return media_type, base64_data def _transform_request( self, @@ -99,15 +132,58 @@ class AmazonNovaEmbeddingConfig: if "embeddingDimension" not in embedding_params: embedding_params["embeddingDimension"] = 3072 - # For text input, add basic text structure if user hasn't provided text/image/video/audio + # For text/media input, add basic structure if user hasn't provided text/image/video/audio if "text" not in embedding_params and "image" not in embedding_params and "video" not in embedding_params and "audio" not in embedding_params: - # Default to text if no modality specified - if input.startswith("s3://"): + # Check if input is a data URL (e.g., data:image/jpeg;base64,...) + if input.startswith("data:"): + # Parse the data URL to extract media type and base64 data + media_type, base64_data = self._parse_data_url(input) + + if media_type.startswith("image/"): + # Extract image format from MIME type (e.g., image/jpeg -> jpeg) + image_format = media_type.split("/")[1].lower() + # Nova API expects specific formats + if image_format == "jpg": + image_format = "jpeg" + + embedding_params["image"] = { + "format": image_format, + "source": { + "bytes": base64_data + } + } + elif media_type.startswith("video/"): + # Handle video data URLs + video_format = media_type.split("/")[1].lower() + embedding_params["video"] = { + "format": video_format, + "source": { + "bytes": base64_data + } + } + elif media_type.startswith("audio/"): + # Handle audio data URLs + audio_format = media_type.split("/")[1].lower() + embedding_params["audio"] = { + "format": audio_format, + "source": { + "bytes": base64_data + } + } + else: + # Fallback to text for unknown types + embedding_params["text"] = { + "value": input, + "truncationMode": "END" + } + elif input.startswith("s3://"): + # S3 URL - default to text for now, user should specify modality embedding_params["text"] = { "source": {"s3Location": {"uri": input}}, "truncationMode": "END" # Required by Nova API } else: + # Plain text input embedding_params["text"] = { "value": input, "truncationMode": "END" # Required by Nova API diff --git a/tests/llm_translation/test_bedrock_nova_embedding.py b/tests/llm_translation/test_bedrock_nova_embedding.py index 8cc77b3c3cb..23064a3389a 100644 --- a/tests/llm_translation/test_bedrock_nova_embedding.py +++ b/tests/llm_translation/test_bedrock_nova_embedding.py @@ -216,6 +216,95 @@ class TestNovaTransformationRequest: params = request["singleEmbeddingParams"] assert params["embeddingDimension"] == 3072 + + def test_data_url_image_parsing(self): + """Test that data URL images are properly parsed and transformed.""" + config = AmazonNovaEmbeddingConfig() + + # Test with JPEG image data URL + jpeg_data_url = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAASABIAAD" + + request = config._transform_request( + input=jpeg_data_url, + inference_params={"dimensions": 1024}, + async_invoke_route=False, + ) + + params = request["singleEmbeddingParams"] + assert "image" in params + assert params["image"]["format"] == "jpeg" + assert "source" in params["image"] + assert params["image"]["source"]["bytes"] == "/9j/4AAQSkZJRgABAQAASABIAAD" + assert params["embeddingDimension"] == 1024 + assert params["embeddingPurpose"] == "GENERIC_INDEX" + + def test_data_url_png_image_parsing(self): + """Test that data URL PNG images are properly parsed.""" + config = AmazonNovaEmbeddingConfig() + + # Test with PNG image data URL + png_data_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ" + + request = config._transform_request( + input=png_data_url, + inference_params={}, + async_invoke_route=False, + ) + + params = request["singleEmbeddingParams"] + assert "image" in params + assert params["image"]["format"] == "png" + assert params["image"]["source"]["bytes"] == "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ" + + def test_data_url_jpg_format_conversion(self): + """Test that jpg format is converted to jpeg.""" + config = AmazonNovaEmbeddingConfig() + + # Test with jpg (should be converted to jpeg) + jpg_data_url = "data:image/jpg;base64,/9j/4AAQSkZJRg" + + request = config._transform_request( + input=jpg_data_url, + inference_params={}, + async_invoke_route=False, + ) + + params = request["singleEmbeddingParams"] + assert params["image"]["format"] == "jpeg" # Should be converted from jpg to jpeg + + def test_data_url_video_parsing(self): + """Test that data URL videos are properly parsed.""" + config = AmazonNovaEmbeddingConfig() + + video_data_url = "data:video/mp4;base64,AAAAIGZ0eXBpc29t" + + request = config._transform_request( + input=video_data_url, + inference_params={}, + async_invoke_route=False, + ) + + params = request["singleEmbeddingParams"] + assert "video" in params + assert params["video"]["format"] == "mp4" + assert params["video"]["source"]["bytes"] == "AAAAIGZ0eXBpc29t" + + def test_data_url_audio_parsing(self): + """Test that data URL audio files are properly parsed.""" + config = AmazonNovaEmbeddingConfig() + + audio_data_url = "data:audio/mp3;base64,SUQzBAAAAAAAI1RTU0UAAAA" + + request = config._transform_request( + input=audio_data_url, + inference_params={}, + async_invoke_route=False, + ) + + params = request["singleEmbeddingParams"] + assert "audio" in params + assert params["audio"]["format"] == "mp3" + assert params["audio"]["source"]["bytes"] == "SUQzBAAAAAAAI1RTU0UAAAA" class TestNovaTransformationResponse: