mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix: request body for image embedding request
This commit is contained in:
Sameer Kankute
parent
46438684ca
commit
cbc1ac9508
3 changed files with 287 additions and 3 deletions
|
|
@ -172,6 +172,125 @@ print(f"Results available at: {output_s3_uri}")
|
|||
|
||||
**Note:** The actual embedding results are stored in S3. When the job is completed, download the results from the S3 location specified in `status.metadata['output_file_id']`. The results will be in JSON/JSONL format containing the embedding vectors.
|
||||
|
||||
## Amazon Nova Multimodal Embeddings
|
||||
|
||||
Amazon Nova supports multimodal embeddings for text, images, video, and audio. It offers flexible embedding dimensions and purposes optimized for different use cases.
|
||||
|
||||
### Supported Features
|
||||
|
||||
- **Modalities**: Text, Image, Video, Audio
|
||||
- **Dimensions**: 256, 384, 1024, 3072 (default: 3072)
|
||||
- **Embedding Purposes**:
|
||||
- `GENERIC_INDEX` (default)
|
||||
- `GENERIC_RETRIEVAL`
|
||||
- `TEXT_RETRIEVAL`
|
||||
- `IMAGE_RETRIEVAL`
|
||||
- `VIDEO_RETRIEVAL`
|
||||
- `AUDIO_RETRIEVAL`
|
||||
- `CLASSIFICATION`
|
||||
- `CLUSTERING`
|
||||
|
||||
### Text Embedding
|
||||
|
||||
```python
|
||||
from litellm import embedding
|
||||
|
||||
response = embedding(
|
||||
model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0",
|
||||
input=["Hello, world!"],
|
||||
aws_region_name="us-east-1",
|
||||
dimensions=1024, # Optional: 256, 384, 1024, or 3072
|
||||
)
|
||||
|
||||
print(response.data[0].embedding)
|
||||
```
|
||||
|
||||
### Image Embedding with Base64
|
||||
|
||||
Amazon Nova accepts images in base64 format using the standard data URL format:
|
||||
|
||||
```python
|
||||
import base64
|
||||
from litellm import embedding
|
||||
|
||||
# Method 1: Load image from file
|
||||
with open("image.jpg", "rb") as image_file:
|
||||
image_data = base64.b64encode(image_file.read()).decode('utf-8')
|
||||
# Create data URL with proper format
|
||||
image_base64 = f"data:image/jpeg;base64,{image_data}"
|
||||
|
||||
response = embedding(
|
||||
model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0",
|
||||
input=[image_base64],
|
||||
aws_region_name="us-east-1",
|
||||
dimensions=1024,
|
||||
)
|
||||
|
||||
print(f"Image embedding: {response.data[0].embedding[:10]}...") # First 10 dimensions
|
||||
```
|
||||
|
||||
#### Supported Image Formats
|
||||
|
||||
Nova supports the following image formats:
|
||||
- JPEG: `data:image/jpeg;base64,...`
|
||||
- PNG: `data:image/png;base64,...`
|
||||
- GIF: `data:image/gif;base64,...`
|
||||
- WebP: `data:image/webp;base64,...`
|
||||
|
||||
#### Complete Example with Error Handling
|
||||
|
||||
```python
|
||||
import base64
|
||||
from litellm import embedding
|
||||
|
||||
def get_image_embedding(image_path, dimensions=1024):
|
||||
"""
|
||||
Get embedding for an image file.
|
||||
|
||||
Args:
|
||||
image_path: Path to the image file
|
||||
dimensions: Embedding dimension (256, 384, 1024, or 3072)
|
||||
|
||||
Returns:
|
||||
List of embedding values
|
||||
"""
|
||||
try:
|
||||
# Determine image format from file extension
|
||||
if image_path.lower().endswith('.png'):
|
||||
mime_type = "image/png"
|
||||
elif image_path.lower().endswith(('.jpg', '.jpeg')):
|
||||
mime_type = "image/jpeg"
|
||||
elif image_path.lower().endswith('.gif'):
|
||||
mime_type = "image/gif"
|
||||
elif image_path.lower().endswith('.webp'):
|
||||
mime_type = "image/webp"
|
||||
else:
|
||||
raise ValueError(f"Unsupported image format: {image_path}")
|
||||
|
||||
# Read and encode image
|
||||
with open(image_path, "rb") as image_file:
|
||||
image_data = base64.b64encode(image_file.read()).decode('utf-8')
|
||||
image_base64 = f"data:{mime_type};base64,{image_data}"
|
||||
|
||||
# Get embedding
|
||||
response = embedding(
|
||||
model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0",
|
||||
input=[image_base64],
|
||||
aws_region_name="us-east-1",
|
||||
dimensions=dimensions,
|
||||
)
|
||||
|
||||
return response.data[0].embedding
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error getting image embedding: {e}")
|
||||
raise
|
||||
|
||||
# Example usage
|
||||
image_embedding = get_image_embedding("photo.jpg", dimensions=1024)
|
||||
print(f"Got embedding with {len(image_embedding)} dimensions")
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
#### Common Errors
|
||||
|
|
|
|||
|
|
@ -46,6 +46,39 @@ class AmazonNovaEmbeddingConfig:
|
|||
elif k in self.get_supported_openai_params():
|
||||
optional_params[k] = v
|
||||
return optional_params
|
||||
|
||||
def _parse_data_url(self, data_url: str) -> tuple:
|
||||
"""
|
||||
Parse a data URL to extract the media type and base64 data.
|
||||
|
||||
Args:
|
||||
data_url: Data URL in format: data:image/jpeg;base64,/9j/4AAQ...
|
||||
|
||||
Returns:
|
||||
tuple: (media_type, base64_data)
|
||||
media_type: e.g., "image/jpeg", "video/mp4", "audio/mpeg"
|
||||
base64_data: The base64-encoded data without the prefix
|
||||
"""
|
||||
if not data_url.startswith("data:"):
|
||||
raise ValueError(f"Invalid data URL format: {data_url[:50]}...")
|
||||
|
||||
# Split by comma to separate metadata from data
|
||||
# Format: data:image/jpeg;base64,<base64_data>
|
||||
if "," not in data_url:
|
||||
raise ValueError(f"Invalid data URL format (missing comma): {data_url[:50]}...")
|
||||
|
||||
metadata, base64_data = data_url.split(",", 1)
|
||||
|
||||
# Extract media type from metadata
|
||||
# Remove 'data:' prefix and ';base64' suffix
|
||||
metadata = metadata[5:] # Remove 'data:'
|
||||
|
||||
if ";" in metadata:
|
||||
media_type = metadata.split(";")[0]
|
||||
else:
|
||||
media_type = metadata
|
||||
|
||||
return media_type, base64_data
|
||||
|
||||
def _transform_request(
|
||||
self,
|
||||
|
|
@ -99,15 +132,58 @@ class AmazonNovaEmbeddingConfig:
|
|||
if "embeddingDimension" not in embedding_params:
|
||||
embedding_params["embeddingDimension"] = 3072
|
||||
|
||||
# For text input, add basic text structure if user hasn't provided text/image/video/audio
|
||||
# For text/media input, add basic structure if user hasn't provided text/image/video/audio
|
||||
if "text" not in embedding_params and "image" not in embedding_params and "video" not in embedding_params and "audio" not in embedding_params:
|
||||
# Default to text if no modality specified
|
||||
if input.startswith("s3://"):
|
||||
# Check if input is a data URL (e.g., data:image/jpeg;base64,...)
|
||||
if input.startswith("data:"):
|
||||
# Parse the data URL to extract media type and base64 data
|
||||
media_type, base64_data = self._parse_data_url(input)
|
||||
|
||||
if media_type.startswith("image/"):
|
||||
# Extract image format from MIME type (e.g., image/jpeg -> jpeg)
|
||||
image_format = media_type.split("/")[1].lower()
|
||||
# Nova API expects specific formats
|
||||
if image_format == "jpg":
|
||||
image_format = "jpeg"
|
||||
|
||||
embedding_params["image"] = {
|
||||
"format": image_format,
|
||||
"source": {
|
||||
"bytes": base64_data
|
||||
}
|
||||
}
|
||||
elif media_type.startswith("video/"):
|
||||
# Handle video data URLs
|
||||
video_format = media_type.split("/")[1].lower()
|
||||
embedding_params["video"] = {
|
||||
"format": video_format,
|
||||
"source": {
|
||||
"bytes": base64_data
|
||||
}
|
||||
}
|
||||
elif media_type.startswith("audio/"):
|
||||
# Handle audio data URLs
|
||||
audio_format = media_type.split("/")[1].lower()
|
||||
embedding_params["audio"] = {
|
||||
"format": audio_format,
|
||||
"source": {
|
||||
"bytes": base64_data
|
||||
}
|
||||
}
|
||||
else:
|
||||
# Fallback to text for unknown types
|
||||
embedding_params["text"] = {
|
||||
"value": input,
|
||||
"truncationMode": "END"
|
||||
}
|
||||
elif input.startswith("s3://"):
|
||||
# S3 URL - default to text for now, user should specify modality
|
||||
embedding_params["text"] = {
|
||||
"source": {"s3Location": {"uri": input}},
|
||||
"truncationMode": "END" # Required by Nova API
|
||||
}
|
||||
else:
|
||||
# Plain text input
|
||||
embedding_params["text"] = {
|
||||
"value": input,
|
||||
"truncationMode": "END" # Required by Nova API
|
||||
|
|
|
|||
|
|
@ -216,6 +216,95 @@ class TestNovaTransformationRequest:
|
|||
|
||||
params = request["singleEmbeddingParams"]
|
||||
assert params["embeddingDimension"] == 3072
|
||||
|
||||
def test_data_url_image_parsing(self):
|
||||
"""Test that data URL images are properly parsed and transformed."""
|
||||
config = AmazonNovaEmbeddingConfig()
|
||||
|
||||
# Test with JPEG image data URL
|
||||
jpeg_data_url = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAASABIAAD"
|
||||
|
||||
request = config._transform_request(
|
||||
input=jpeg_data_url,
|
||||
inference_params={"dimensions": 1024},
|
||||
async_invoke_route=False,
|
||||
)
|
||||
|
||||
params = request["singleEmbeddingParams"]
|
||||
assert "image" in params
|
||||
assert params["image"]["format"] == "jpeg"
|
||||
assert "source" in params["image"]
|
||||
assert params["image"]["source"]["bytes"] == "/9j/4AAQSkZJRgABAQAASABIAAD"
|
||||
assert params["embeddingDimension"] == 1024
|
||||
assert params["embeddingPurpose"] == "GENERIC_INDEX"
|
||||
|
||||
def test_data_url_png_image_parsing(self):
|
||||
"""Test that data URL PNG images are properly parsed."""
|
||||
config = AmazonNovaEmbeddingConfig()
|
||||
|
||||
# Test with PNG image data URL
|
||||
png_data_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ"
|
||||
|
||||
request = config._transform_request(
|
||||
input=png_data_url,
|
||||
inference_params={},
|
||||
async_invoke_route=False,
|
||||
)
|
||||
|
||||
params = request["singleEmbeddingParams"]
|
||||
assert "image" in params
|
||||
assert params["image"]["format"] == "png"
|
||||
assert params["image"]["source"]["bytes"] == "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ"
|
||||
|
||||
def test_data_url_jpg_format_conversion(self):
|
||||
"""Test that jpg format is converted to jpeg."""
|
||||
config = AmazonNovaEmbeddingConfig()
|
||||
|
||||
# Test with jpg (should be converted to jpeg)
|
||||
jpg_data_url = "data:image/jpg;base64,/9j/4AAQSkZJRg"
|
||||
|
||||
request = config._transform_request(
|
||||
input=jpg_data_url,
|
||||
inference_params={},
|
||||
async_invoke_route=False,
|
||||
)
|
||||
|
||||
params = request["singleEmbeddingParams"]
|
||||
assert params["image"]["format"] == "jpeg" # Should be converted from jpg to jpeg
|
||||
|
||||
def test_data_url_video_parsing(self):
|
||||
"""Test that data URL videos are properly parsed."""
|
||||
config = AmazonNovaEmbeddingConfig()
|
||||
|
||||
video_data_url = "data:video/mp4;base64,AAAAIGZ0eXBpc29t"
|
||||
|
||||
request = config._transform_request(
|
||||
input=video_data_url,
|
||||
inference_params={},
|
||||
async_invoke_route=False,
|
||||
)
|
||||
|
||||
params = request["singleEmbeddingParams"]
|
||||
assert "video" in params
|
||||
assert params["video"]["format"] == "mp4"
|
||||
assert params["video"]["source"]["bytes"] == "AAAAIGZ0eXBpc29t"
|
||||
|
||||
def test_data_url_audio_parsing(self):
|
||||
"""Test that data URL audio files are properly parsed."""
|
||||
config = AmazonNovaEmbeddingConfig()
|
||||
|
||||
audio_data_url = "data:audio/mp3;base64,SUQzBAAAAAAAI1RTU0UAAAA"
|
||||
|
||||
request = config._transform_request(
|
||||
input=audio_data_url,
|
||||
inference_params={},
|
||||
async_invoke_route=False,
|
||||
)
|
||||
|
||||
params = request["singleEmbeddingParams"]
|
||||
assert "audio" in params
|
||||
assert params["audio"]["format"] == "mp3"
|
||||
assert params["audio"]["source"]["bytes"] == "SUQzBAAAAAAAI1RTU0UAAAA"
|
||||
|
||||
|
||||
class TestNovaTransformationResponse:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue