feat(bedrock): support combined text+image embedding for Titan and Nova multimodal models

Agent-Logs-Url: https://github.com/evercompliant/litellm/sessions/ec6eb8b5-2105-4067-b55f-32c0e9cab805

Co-authored-by: serhiimeverc <106583451+serhiimeverc@users.noreply.github.com>

docs/my-website/docs/embedding/supported_embedding.md

@@ -158,6 +158,96 @@ curl -X POST 'http://0.0.0.0:4000/v1/embeddings' \
 </TabItem>
 </Tabs>
 
+## Combined Text + Image Embeddings (Bedrock)
+
+Bedrock multimodal embedding models (`amazon.titan-embed-image-v1` and `amazon.nova-2-multimodal-embeddings-v1:0`) support sending both text and image together in a single embedding request.
+
+Pass the text string **immediately before** its paired image in the `input` list. LiteLLM will detect the adjacent `[text, base64_image]` pair and merge them into a single Bedrock request containing both `inputText` and `inputImage` (Titan) or both `text` and `image` (Nova).
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import base64
+from litellm import embedding
+
+with open("product.jpg", "rb") as f:
+    image_b64 = base64.b64encode(f.read()).decode("utf-8")
+
+# Titan multimodal — text + image in one request
+response = embedding(
+    model="bedrock/amazon.titan-embed-image-v1",
+    input=[
+        "Red leather handbag with gold buckle",       # text
+        f"data:image/jpeg;base64,{image_b64}",        # image (immediately after text)
+    ],
+    dimensions=1024,
+    aws_region_name="us-east-1",
+)
+
+# Nova multimodal — same pattern
+response = embedding(
+    model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0",
+    input=[
+        "shoes photo",                                # text
+        f"data:image/jpeg;base64,{image_b64}",        # image (immediately after text)
+    ],
+    aws_region_name="us-east-1",
+)
+```
+
+Single-modality inputs continue to work unchanged:
+
+```python
+# Text only
+response = embedding(model="bedrock/amazon.titan-embed-image-v1", input=["some text"])
+
+# Image only
+response = embedding(
+    model="bedrock/amazon.titan-embed-image-v1",
+    input=[f"data:image/jpeg;base64,{image_b64}"],
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: titan-multimodal
+    litellm_params:
+      model: bedrock/amazon.titan-embed-image-v1
+      aws_region_name: us-east-1
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it — pass text immediately before its paired image:
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/v1/embeddings' \
+  -H 'Authorization: Bearer <your-key>' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "titan-multimodal",
+    "input": [
+      "Red leather handbag with gold buckle",
+      "data:image/jpeg;base64,<base64_image_string>"
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
 ## Input Params for `litellm.embedding()`
 
 

litellm/llms/bedrock/embed/amazon_nova_transformation.py

@@ -94,6 +94,7 @@ class AmazonNovaEmbeddingConfig:
         async_invoke_route: bool = False,
         model_id: Optional[str] = None,
         output_s3_uri: Optional[str] = None,
+        input_text: Optional[str] = None,
     ) -> dict:
         """
         Transform OpenAI-style input to Nova format.
@@ -164,6 +165,12 @@ class AmazonNovaEmbeddingConfig:
                         "format": image_format,
                         "source": {"bytes": base64_data},
                     }
+                    # If paired text was provided, include it alongside the image
+                    if input_text is not None:
+                        embedding_params["text"] = {
+                            "value": input_text,
+                            "truncationMode": "END",
+                        }
                 elif media_type.startswith("video/"):
                     # Handle video data URLs
                     video_format = media_type.split("/")[1].lower()

litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py

@@ -44,7 +44,10 @@ class AmazonTitanMultimodalEmbeddingG1Config:
         return optional_params
 
     def _transform_request(
-        self, input: str, inference_params: dict
+        self,
+        input: str,
+        inference_params: dict,
+        input_text: Optional[str] = None,
     ) -> AmazonTitanMultimodalEmbeddingRequest:
         ## check if b64 encoded str or not ##
         is_encoded = is_base64_encoded(input)
@@ -53,6 +56,9 @@ class AmazonTitanMultimodalEmbeddingG1Config:
             transformed_request = AmazonTitanMultimodalEmbeddingRequest(
                 inputImage=b64_str
             )
+            # If paired text was provided, include it alongside the image
+            if input_text is not None:
+                transformed_request["inputText"] = input_text
         else:
             transformed_request = AmazonTitanMultimodalEmbeddingRequest(inputText=input)
 

litellm/llms/bedrock/embed/embedding.py

@@ -24,6 +24,7 @@ from litellm.types.llms.bedrock import (
     CohereEmbeddingRequest,
 )
 from litellm.types.utils import EmbeddingResponse, LlmProviders
+from litellm.utils import is_base64_encoded
 
 from ..base_aws_llm import BaseAWSLLM
 from ..common_utils import BedrockError
@@ -449,33 +450,57 @@ class BedrockEmbedding(BaseAWSLLM):
             "amazon.titan-embed-text-v2:0",
         ]:
             batch_data = []
-            for i in input:
-                if model == "amazon.titan-embed-image-v1":
-                    transformed_request: (
-                        AmazonEmbeddingRequest
-                    ) = AmazonTitanMultimodalEmbeddingG1Config()._transform_request(
-                        input=i, inference_params=inference_params
-                    )
-                elif model == "amazon.titan-embed-text-v1":
-                    transformed_request = AmazonTitanG1Config()._transform_request(
-                        input=i, inference_params=inference_params
-                    )
-                elif model == "amazon.titan-embed-text-v2:0":
-                    transformed_request = AmazonTitanV2Config()._transform_request(
-                        input=i, inference_params=inference_params
-                    )
-                else:
-                    raise Exception(
-                        "Unmapped model. Received={}. Expected={}".format(
-                            model,
-                            [
-                                "amazon.titan-embed-image-v1",
-                                "amazon.titan-embed-text-v1",
-                                "amazon.titan-embed-text-v2:0",
-                            ],
+            if model == "amazon.titan-embed-image-v1":
+                # Scan for adjacent [text, base64_image] pairs and combine them
+                idx = 0
+                while idx < len(input):
+                    current = input[idx]
+                    next_elem = input[idx + 1] if idx + 1 < len(input) else None
+                    if (
+                        isinstance(current, str)
+                        and not is_base64_encoded(current)
+                        and next_elem is not None
+                        and is_base64_encoded(next_elem)
+                    ):
+                        # Text followed by image → combined request
+                        transformed_request: (
+                            AmazonEmbeddingRequest
+                        ) = AmazonTitanMultimodalEmbeddingG1Config()._transform_request(
+                            input=next_elem,
+                            inference_params=inference_params,
+                            input_text=current,
                         )
-                    )
-                batch_data.append(transformed_request)
+                        batch_data.append(transformed_request)
+                        idx += 2
+                    else:
+                        transformed_request = (
+                            AmazonTitanMultimodalEmbeddingG1Config()._transform_request(
+                                input=current, inference_params=inference_params
+                            )
+                        )
+                        batch_data.append(transformed_request)
+                        idx += 1
+            else:
+                for i in input:
+                    if model == "amazon.titan-embed-text-v1":
+                        transformed_request = AmazonTitanG1Config()._transform_request(
+                            input=i, inference_params=inference_params
+                        )
+                    elif model == "amazon.titan-embed-text-v2:0":
+                        transformed_request = AmazonTitanV2Config()._transform_request(
+                            input=i, inference_params=inference_params
+                        )
+                    else:
+                        raise Exception(
+                            "Unmapped model. Received={}. Expected={}".format(
+                                model,
+                                [
+                                    "amazon.titan-embed-text-v1",
+                                    "amazon.titan-embed-text-v2:0",
+                                ],
+                            )
+                        )
+                    batch_data.append(transformed_request)
         elif provider == "twelvelabs":
             batch_data = []
             for i in input:
@@ -491,15 +516,39 @@ class BedrockEmbedding(BaseAWSLLM):
                 batch_data.append(twelvelabs_request)
         elif provider == "nova":
             batch_data = []
-            for i in input:
-                nova_request = AmazonNovaEmbeddingConfig()._transform_request(
-                    input=i,
-                    inference_params=inference_params,
-                    async_invoke_route=has_async_invoke,
-                    model_id=modelId,
-                    output_s3_uri=inference_params.get("output_s3_uri"),
-                )
-                batch_data.append(nova_request)
+            # Scan for adjacent [text, data:image/...] pairs and combine them
+            idx = 0
+            while idx < len(input):
+                current = input[idx]
+                next_elem = input[idx + 1] if idx + 1 < len(input) else None
+                if (
+                    isinstance(current, str)
+                    and not current.startswith("data:")
+                    and next_elem is not None
+                    and isinstance(next_elem, str)
+                    and next_elem.startswith("data:image/")
+                ):
+                    # Text followed by image → combined request
+                    nova_request = AmazonNovaEmbeddingConfig()._transform_request(
+                        input=next_elem,
+                        inference_params=inference_params,
+                        async_invoke_route=has_async_invoke,
+                        model_id=modelId,
+                        output_s3_uri=inference_params.get("output_s3_uri"),
+                        input_text=current,
+                    )
+                    batch_data.append(nova_request)
+                    idx += 2
+                else:
+                    nova_request = AmazonNovaEmbeddingConfig()._transform_request(
+                        input=current,
+                        inference_params=inference_params,
+                        async_invoke_route=has_async_invoke,
+                        model_id=modelId,
+                        output_s3_uri=inference_params.get("output_s3_uri"),
+                    )
+                    batch_data.append(nova_request)
+                    idx += 1
 
         ### SET RUNTIME ENDPOINT ###
         endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(

tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py

@@ -955,3 +955,199 @@ def test_titan_image_embedding_cost_uses_per_image_rate():
         assert response.usage is not None
         assert response.usage.prompt_tokens_details is not None
         assert response.usage.prompt_tokens_details.image_count == 1
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Combined text + image embedding tests (Titan multimodal)
+# ──────────────────────────────────────────────────────────────────────────────
+
+
+def test_titan_multimodal_transform_request_combined():
+    """Unit test: _transform_request with both input (image) and input_text produces both keys."""
+    from litellm.llms.bedrock.embed.amazon_titan_multimodal_transformation import (
+        AmazonTitanMultimodalEmbeddingG1Config,
+    )
+
+    config = AmazonTitanMultimodalEmbeddingG1Config()
+    result = config._transform_request(
+        input='data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==',
+        inference_params={},
+        input_text='A red handbag',
+    )
+
+    assert 'inputImage' in result
+    assert 'inputText' in result
+    assert result['inputText'] == 'A red handbag'
+
+
+def test_titan_multimodal_transform_request_image_only_no_text():
+    """Unit test: _transform_request with image only (no input_text) produces only inputImage."""
+    from litellm.llms.bedrock.embed.amazon_titan_multimodal_transformation import (
+        AmazonTitanMultimodalEmbeddingG1Config,
+    )
+
+    config = AmazonTitanMultimodalEmbeddingG1Config()
+    result = config._transform_request(
+        input='data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==',
+        inference_params={},
+    )
+
+    assert 'inputImage' in result
+    assert 'inputText' not in result
+
+
+def test_titan_multimodal_combined_text_and_image_request():
+    """Integration test: passing [text, image] list produces one HTTP call with both keys."""
+    client = HTTPHandler()
+    embed_response = {
+        'embedding': [0.1, 0.2, 0.3],
+        'inputTextTokenCount': 5,
+    }
+
+    with patch.object(client, 'post') as mock_post:
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = json.dumps(embed_response)
+        mock_response.json = lambda: json.loads(mock_response.text)
+        mock_post.return_value = mock_response
+
+        response = litellm.embedding(
+            model='bedrock/amazon.titan-embed-image-v1',
+            input=[
+                'Red leather handbag',
+                'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==',
+            ],
+            client=client,
+            aws_access_key_id='fake',
+            aws_secret_access_key='fake',
+            aws_region_name='us-east-1',
+        )
+
+        assert isinstance(response, litellm.EmbeddingResponse)
+        # Only one HTTP call should have been made (text+image merged)
+        assert mock_post.call_count == 1
+
+        request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}'))
+        assert 'inputText' in request_body
+        assert 'inputImage' in request_body
+        assert request_body['inputText'] == 'Red leather handbag'
+
+
+def test_titan_multimodal_text_only_unchanged():
+    """Backward compat: text-only input still produces only inputText."""
+    client = HTTPHandler()
+    embed_response = {
+        'embedding': [0.1, 0.2, 0.3],
+        'inputTextTokenCount': 5,
+    }
+
+    with patch.object(client, 'post') as mock_post:
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = json.dumps(embed_response)
+        mock_response.json = lambda: json.loads(mock_response.text)
+        mock_post.return_value = mock_response
+
+        response = litellm.embedding(
+            model='bedrock/amazon.titan-embed-image-v1',
+            input=['just some text'],
+            client=client,
+            aws_access_key_id='fake',
+            aws_secret_access_key='fake',
+            aws_region_name='us-east-1',
+        )
+
+        assert isinstance(response, litellm.EmbeddingResponse)
+        request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}'))
+        assert 'inputText' in request_body
+        assert 'inputImage' not in request_body
+
+
+def test_titan_multimodal_image_only_unchanged():
+    """Backward compat: image-only input still produces only inputImage."""
+    client = HTTPHandler()
+    embed_response = {
+        'embedding': [0.1, 0.2, 0.3],
+        'inputTextTokenCount': 0,
+    }
+
+    with patch.object(client, 'post') as mock_post:
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = json.dumps(embed_response)
+        mock_response.json = lambda: json.loads(mock_response.text)
+        mock_post.return_value = mock_response
+
+        response = litellm.embedding(
+            model='bedrock/amazon.titan-embed-image-v1',
+            input=['data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=='],
+            client=client,
+            aws_access_key_id='fake',
+            aws_secret_access_key='fake',
+            aws_region_name='us-east-1',
+        )
+
+        assert isinstance(response, litellm.EmbeddingResponse)
+        request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}'))
+        assert 'inputImage' in request_body
+        assert 'inputText' not in request_body
+
+
+# ──────────────────────────────────────────────────────────────────────────────
+# Combined text + image embedding tests (Nova multimodal)
+# ──────────────────────────────────────────────────────────────────────────────
+
+
+def test_nova_combined_text_and_image_request():
+    """Unit test: Nova _transform_request with input_text alongside image produces both keys."""
+    from litellm.llms.bedrock.embed.amazon_nova_transformation import (
+        AmazonNovaEmbeddingConfig,
+    )
+
+    config = AmazonNovaEmbeddingConfig()
+    result = config._transform_request(
+        input='data:image/jpeg;base64,/9j/4AAQSkZJRgAB',
+        inference_params={},
+        input_text='shoes photo',
+    )
+
+    params = result.get('singleEmbeddingParams', {})
+    assert 'image' in params
+    assert 'text' in params
+    assert params['text']['value'] == 'shoes photo'
+
+
+def test_nova_combined_text_and_image_http_request():
+    """Integration test: passing [text, image] list to Nova produces one call with both keys."""
+    client = HTTPHandler()
+    nova_response = {
+        'embeddings': [
+            {'embeddingType': 'IMAGE', 'embedding': [0.1, 0.2, 0.3]},
+        ]
+    }
+
+    with patch.object(client, 'post') as mock_post:
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.text = json.dumps(nova_response)
+        mock_response.json = lambda: json.loads(mock_response.text)
+        mock_post.return_value = mock_response
+
+        response = litellm.embedding(
+            model='bedrock/amazon.nova-2-multimodal-embeddings-v1:0',
+            input=['shoes photo', 'data:image/jpeg;base64,/9j/4AAQSkZJRgAB'],
+            client=client,
+            aws_access_key_id='fake',
+            aws_secret_access_key='fake',
+            aws_region_name='us-east-1',
+        )
+
+        assert isinstance(response, litellm.EmbeddingResponse)
+        # Only one HTTP call (text+image merged)
+        assert mock_post.call_count == 1
+
+        request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}'))
+        params = request_body.get('singleEmbeddingParams', {})
+        assert 'image' in params
+        assert 'text' in params
+        assert params['text']['value'] == 'shoes photo'