diff --git a/docs/my-website/docs/embedding/supported_embedding.md b/docs/my-website/docs/embedding/supported_embedding.md index 87acd0b33a5..0f5544132ad 100644 --- a/docs/my-website/docs/embedding/supported_embedding.md +++ b/docs/my-website/docs/embedding/supported_embedding.md @@ -158,6 +158,96 @@ curl -X POST 'http://0.0.0.0:4000/v1/embeddings' \ +## Combined Text + Image Embeddings (Bedrock) + +Bedrock multimodal embedding models (`amazon.titan-embed-image-v1` and `amazon.nova-2-multimodal-embeddings-v1:0`) support sending both text and image together in a single embedding request. + +Pass the text string **immediately before** its paired image in the `input` list. LiteLLM will detect the adjacent `[text, base64_image]` pair and merge them into a single Bedrock request containing both `inputText` and `inputImage` (Titan) or both `text` and `image` (Nova). + + + + +```python +import base64 +from litellm import embedding + +with open("product.jpg", "rb") as f: + image_b64 = base64.b64encode(f.read()).decode("utf-8") + +# Titan multimodal — text + image in one request +response = embedding( + model="bedrock/amazon.titan-embed-image-v1", + input=[ + "Red leather handbag with gold buckle", # text + f"data:image/jpeg;base64,{image_b64}", # image (immediately after text) + ], + dimensions=1024, + aws_region_name="us-east-1", +) + +# Nova multimodal — same pattern +response = embedding( + model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", + input=[ + "shoes photo", # text + f"data:image/jpeg;base64,{image_b64}", # image (immediately after text) + ], + aws_region_name="us-east-1", +) +``` + +Single-modality inputs continue to work unchanged: + +```python +# Text only +response = embedding(model="bedrock/amazon.titan-embed-image-v1", input=["some text"]) + +# Image only +response = embedding( + model="bedrock/amazon.titan-embed-image-v1", + input=[f"data:image/jpeg;base64,{image_b64}"], +) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: titan-multimodal + litellm_params: + model: bedrock/amazon.titan-embed-image-v1 + aws_region_name: us-east-1 +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +3. Test it — pass text immediately before its paired image: + +```bash +curl -X POST 'http://0.0.0.0:4000/v1/embeddings' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "titan-multimodal", + "input": [ + "Red leather handbag with gold buckle", + "data:image/jpeg;base64," + ] + }' +``` + + + + ## Input Params for `litellm.embedding()` diff --git a/litellm/llms/bedrock/embed/amazon_nova_transformation.py b/litellm/llms/bedrock/embed/amazon_nova_transformation.py index c20b52a6e0d..5fbe6490731 100644 --- a/litellm/llms/bedrock/embed/amazon_nova_transformation.py +++ b/litellm/llms/bedrock/embed/amazon_nova_transformation.py @@ -94,6 +94,7 @@ class AmazonNovaEmbeddingConfig: async_invoke_route: bool = False, model_id: Optional[str] = None, output_s3_uri: Optional[str] = None, + input_text: Optional[str] = None, ) -> dict: """ Transform OpenAI-style input to Nova format. @@ -164,6 +165,12 @@ class AmazonNovaEmbeddingConfig: "format": image_format, "source": {"bytes": base64_data}, } + # If paired text was provided, include it alongside the image + if input_text is not None: + embedding_params["text"] = { + "value": input_text, + "truncationMode": "END", + } elif media_type.startswith("video/"): # Handle video data URLs video_format = media_type.split("/")[1].lower() diff --git a/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py b/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py index 07b04734c30..8e0aad0cfdd 100644 --- a/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py +++ b/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py @@ -44,7 +44,10 @@ class AmazonTitanMultimodalEmbeddingG1Config: return optional_params def _transform_request( - self, input: str, inference_params: dict + self, + input: str, + inference_params: dict, + input_text: Optional[str] = None, ) -> AmazonTitanMultimodalEmbeddingRequest: ## check if b64 encoded str or not ## is_encoded = is_base64_encoded(input) @@ -53,6 +56,9 @@ class AmazonTitanMultimodalEmbeddingG1Config: transformed_request = AmazonTitanMultimodalEmbeddingRequest( inputImage=b64_str ) + # If paired text was provided, include it alongside the image + if input_text is not None: + transformed_request["inputText"] = input_text else: transformed_request = AmazonTitanMultimodalEmbeddingRequest(inputText=input) diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 27dc785bf57..5a82c65b9c4 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -24,6 +24,7 @@ from litellm.types.llms.bedrock import ( CohereEmbeddingRequest, ) from litellm.types.utils import EmbeddingResponse, LlmProviders +from litellm.utils import is_base64_encoded from ..base_aws_llm import BaseAWSLLM from ..common_utils import BedrockError @@ -449,33 +450,57 @@ class BedrockEmbedding(BaseAWSLLM): "amazon.titan-embed-text-v2:0", ]: batch_data = [] - for i in input: - if model == "amazon.titan-embed-image-v1": - transformed_request: ( - AmazonEmbeddingRequest - ) = AmazonTitanMultimodalEmbeddingG1Config()._transform_request( - input=i, inference_params=inference_params - ) - elif model == "amazon.titan-embed-text-v1": - transformed_request = AmazonTitanG1Config()._transform_request( - input=i, inference_params=inference_params - ) - elif model == "amazon.titan-embed-text-v2:0": - transformed_request = AmazonTitanV2Config()._transform_request( - input=i, inference_params=inference_params - ) - else: - raise Exception( - "Unmapped model. Received={}. Expected={}".format( - model, - [ - "amazon.titan-embed-image-v1", - "amazon.titan-embed-text-v1", - "amazon.titan-embed-text-v2:0", - ], + if model == "amazon.titan-embed-image-v1": + # Scan for adjacent [text, base64_image] pairs and combine them + idx = 0 + while idx < len(input): + current = input[idx] + next_elem = input[idx + 1] if idx + 1 < len(input) else None + if ( + isinstance(current, str) + and not is_base64_encoded(current) + and next_elem is not None + and is_base64_encoded(next_elem) + ): + # Text followed by image → combined request + transformed_request: ( + AmazonEmbeddingRequest + ) = AmazonTitanMultimodalEmbeddingG1Config()._transform_request( + input=next_elem, + inference_params=inference_params, + input_text=current, ) - ) - batch_data.append(transformed_request) + batch_data.append(transformed_request) + idx += 2 + else: + transformed_request = ( + AmazonTitanMultimodalEmbeddingG1Config()._transform_request( + input=current, inference_params=inference_params + ) + ) + batch_data.append(transformed_request) + idx += 1 + else: + for i in input: + if model == "amazon.titan-embed-text-v1": + transformed_request = AmazonTitanG1Config()._transform_request( + input=i, inference_params=inference_params + ) + elif model == "amazon.titan-embed-text-v2:0": + transformed_request = AmazonTitanV2Config()._transform_request( + input=i, inference_params=inference_params + ) + else: + raise Exception( + "Unmapped model. Received={}. Expected={}".format( + model, + [ + "amazon.titan-embed-text-v1", + "amazon.titan-embed-text-v2:0", + ], + ) + ) + batch_data.append(transformed_request) elif provider == "twelvelabs": batch_data = [] for i in input: @@ -491,15 +516,39 @@ class BedrockEmbedding(BaseAWSLLM): batch_data.append(twelvelabs_request) elif provider == "nova": batch_data = [] - for i in input: - nova_request = AmazonNovaEmbeddingConfig()._transform_request( - input=i, - inference_params=inference_params, - async_invoke_route=has_async_invoke, - model_id=modelId, - output_s3_uri=inference_params.get("output_s3_uri"), - ) - batch_data.append(nova_request) + # Scan for adjacent [text, data:image/...] pairs and combine them + idx = 0 + while idx < len(input): + current = input[idx] + next_elem = input[idx + 1] if idx + 1 < len(input) else None + if ( + isinstance(current, str) + and not current.startswith("data:") + and next_elem is not None + and isinstance(next_elem, str) + and next_elem.startswith("data:image/") + ): + # Text followed by image → combined request + nova_request = AmazonNovaEmbeddingConfig()._transform_request( + input=next_elem, + inference_params=inference_params, + async_invoke_route=has_async_invoke, + model_id=modelId, + output_s3_uri=inference_params.get("output_s3_uri"), + input_text=current, + ) + batch_data.append(nova_request) + idx += 2 + else: + nova_request = AmazonNovaEmbeddingConfig()._transform_request( + input=current, + inference_params=inference_params, + async_invoke_route=has_async_invoke, + model_id=modelId, + output_s3_uri=inference_params.get("output_s3_uri"), + ) + batch_data.append(nova_request) + idx += 1 ### SET RUNTIME ENDPOINT ### endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py index a38a6612f79..a332541d9a2 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py @@ -955,3 +955,199 @@ def test_titan_image_embedding_cost_uses_per_image_rate(): assert response.usage is not None assert response.usage.prompt_tokens_details is not None assert response.usage.prompt_tokens_details.image_count == 1 + + +# ────────────────────────────────────────────────────────────────────────────── +# Combined text + image embedding tests (Titan multimodal) +# ────────────────────────────────────────────────────────────────────────────── + + +def test_titan_multimodal_transform_request_combined(): + """Unit test: _transform_request with both input (image) and input_text produces both keys.""" + from litellm.llms.bedrock.embed.amazon_titan_multimodal_transformation import ( + AmazonTitanMultimodalEmbeddingG1Config, + ) + + config = AmazonTitanMultimodalEmbeddingG1Config() + result = config._transform_request( + input='data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==', + inference_params={}, + input_text='A red handbag', + ) + + assert 'inputImage' in result + assert 'inputText' in result + assert result['inputText'] == 'A red handbag' + + +def test_titan_multimodal_transform_request_image_only_no_text(): + """Unit test: _transform_request with image only (no input_text) produces only inputImage.""" + from litellm.llms.bedrock.embed.amazon_titan_multimodal_transformation import ( + AmazonTitanMultimodalEmbeddingG1Config, + ) + + config = AmazonTitanMultimodalEmbeddingG1Config() + result = config._transform_request( + input='data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==', + inference_params={}, + ) + + assert 'inputImage' in result + assert 'inputText' not in result + + +def test_titan_multimodal_combined_text_and_image_request(): + """Integration test: passing [text, image] list produces one HTTP call with both keys.""" + client = HTTPHandler() + embed_response = { + 'embedding': [0.1, 0.2, 0.3], + 'inputTextTokenCount': 5, + } + + with patch.object(client, 'post') as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(embed_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + response = litellm.embedding( + model='bedrock/amazon.titan-embed-image-v1', + input=[ + 'Red leather handbag', + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==', + ], + client=client, + aws_access_key_id='fake', + aws_secret_access_key='fake', + aws_region_name='us-east-1', + ) + + assert isinstance(response, litellm.EmbeddingResponse) + # Only one HTTP call should have been made (text+image merged) + assert mock_post.call_count == 1 + + request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}')) + assert 'inputText' in request_body + assert 'inputImage' in request_body + assert request_body['inputText'] == 'Red leather handbag' + + +def test_titan_multimodal_text_only_unchanged(): + """Backward compat: text-only input still produces only inputText.""" + client = HTTPHandler() + embed_response = { + 'embedding': [0.1, 0.2, 0.3], + 'inputTextTokenCount': 5, + } + + with patch.object(client, 'post') as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(embed_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + response = litellm.embedding( + model='bedrock/amazon.titan-embed-image-v1', + input=['just some text'], + client=client, + aws_access_key_id='fake', + aws_secret_access_key='fake', + aws_region_name='us-east-1', + ) + + assert isinstance(response, litellm.EmbeddingResponse) + request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}')) + assert 'inputText' in request_body + assert 'inputImage' not in request_body + + +def test_titan_multimodal_image_only_unchanged(): + """Backward compat: image-only input still produces only inputImage.""" + client = HTTPHandler() + embed_response = { + 'embedding': [0.1, 0.2, 0.3], + 'inputTextTokenCount': 0, + } + + with patch.object(client, 'post') as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(embed_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + response = litellm.embedding( + model='bedrock/amazon.titan-embed-image-v1', + input=['data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=='], + client=client, + aws_access_key_id='fake', + aws_secret_access_key='fake', + aws_region_name='us-east-1', + ) + + assert isinstance(response, litellm.EmbeddingResponse) + request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}')) + assert 'inputImage' in request_body + assert 'inputText' not in request_body + + +# ────────────────────────────────────────────────────────────────────────────── +# Combined text + image embedding tests (Nova multimodal) +# ────────────────────────────────────────────────────────────────────────────── + + +def test_nova_combined_text_and_image_request(): + """Unit test: Nova _transform_request with input_text alongside image produces both keys.""" + from litellm.llms.bedrock.embed.amazon_nova_transformation import ( + AmazonNovaEmbeddingConfig, + ) + + config = AmazonNovaEmbeddingConfig() + result = config._transform_request( + input='data:image/jpeg;base64,/9j/4AAQSkZJRgAB', + inference_params={}, + input_text='shoes photo', + ) + + params = result.get('singleEmbeddingParams', {}) + assert 'image' in params + assert 'text' in params + assert params['text']['value'] == 'shoes photo' + + +def test_nova_combined_text_and_image_http_request(): + """Integration test: passing [text, image] list to Nova produces one call with both keys.""" + client = HTTPHandler() + nova_response = { + 'embeddings': [ + {'embeddingType': 'IMAGE', 'embedding': [0.1, 0.2, 0.3]}, + ] + } + + with patch.object(client, 'post') as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(nova_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + response = litellm.embedding( + model='bedrock/amazon.nova-2-multimodal-embeddings-v1:0', + input=['shoes photo', 'data:image/jpeg;base64,/9j/4AAQSkZJRgAB'], + client=client, + aws_access_key_id='fake', + aws_secret_access_key='fake', + aws_region_name='us-east-1', + ) + + assert isinstance(response, litellm.EmbeddingResponse) + # Only one HTTP call (text+image merged) + assert mock_post.call_count == 1 + + request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}')) + params = request_body.get('singleEmbeddingParams', {}) + assert 'image' in params + assert 'text' in params + assert params['text']['value'] == 'shoes photo'