From 4f95bfc967438760aec1e86f50099d2fcd4fc877 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Apr 2026 14:08:07 +0000 Subject: [PATCH 1/2] Initial plan From 0f37e4729240f8233724551bc136accaaf09e1fb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 14 Apr 2026 14:15:57 +0000 Subject: [PATCH 2/2] remediation: add combine_text_image_pairs opt-in flag, fix image detection, add coverage tests Agent-Logs-Url: https://github.com/evercompliant/litellm/sessions/0d331a54-d795-4a92-a7b8-b869c0ef66b0 Co-authored-by: serhiimeverc <106583451+serhiimeverc@users.noreply.github.com> --- .../docs/embedding/supported_embedding.md | 7 +- litellm/llms/bedrock/embed/embedding.py | 127 ++++++++------- .../bedrock/embed/test_bedrock_embedding.py | 144 ++++++++++++++++++ 3 files changed, 226 insertions(+), 52 deletions(-) diff --git a/docs/my-website/docs/embedding/supported_embedding.md b/docs/my-website/docs/embedding/supported_embedding.md index 0f5544132ad..91077068f85 100644 --- a/docs/my-website/docs/embedding/supported_embedding.md +++ b/docs/my-website/docs/embedding/supported_embedding.md @@ -162,7 +162,9 @@ curl -X POST 'http://0.0.0.0:4000/v1/embeddings' \ Bedrock multimodal embedding models (`amazon.titan-embed-image-v1` and `amazon.nova-2-multimodal-embeddings-v1:0`) support sending both text and image together in a single embedding request. -Pass the text string **immediately before** its paired image in the `input` list. LiteLLM will detect the adjacent `[text, base64_image]` pair and merge them into a single Bedrock request containing both `inputText` and `inputImage` (Titan) or both `text` and `image` (Nova). +Pass the text string **immediately before** its paired image in the `input` list and set `combine_text_image_pairs=True`. LiteLLM will detect the adjacent `[text, base64_image]` pair and merge them into a single Bedrock request containing both `inputText` and `inputImage` (Titan) or both `text` and `image` (Nova). + +When `combine_text_image_pairs` is omitted or `False` (the default), each element in `input` is sent as a separate request — preserving backward compatibility for existing code. @@ -181,6 +183,7 @@ response = embedding( "Red leather handbag with gold buckle", # text f"data:image/jpeg;base64,{image_b64}", # image (immediately after text) ], + combine_text_image_pairs=True, dimensions=1024, aws_region_name="us-east-1", ) @@ -192,6 +195,7 @@ response = embedding( "shoes photo", # text f"data:image/jpeg;base64,{image_b64}", # image (immediately after text) ], + combine_text_image_pairs=True, aws_region_name="us-east-1", ) ``` @@ -220,6 +224,7 @@ model_list: litellm_params: model: bedrock/amazon.titan-embed-image-v1 aws_region_name: us-east-1 + combine_text_image_pairs: true ``` 2. Start proxy diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 5a82c65b9c4..411c6c04d14 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -24,7 +24,6 @@ from litellm.types.llms.bedrock import ( CohereEmbeddingRequest, ) from litellm.types.utils import EmbeddingResponse, LlmProviders -from litellm.utils import is_base64_encoded from ..base_aws_llm import BaseAWSLLM from ..common_utils import BedrockError @@ -437,6 +436,9 @@ class BedrockEmbedding(BaseAWSLLM): inference_params.pop( "user", None ) # make sure user is not passed in for bedrock call + combine_text_image_pairs: bool = inference_params.pop( + "combine_text_image_pairs", False + ) data: Optional[CohereEmbeddingRequest] = None batch_data: Optional[List] = None @@ -451,35 +453,46 @@ class BedrockEmbedding(BaseAWSLLM): ]: batch_data = [] if model == "amazon.titan-embed-image-v1": - # Scan for adjacent [text, base64_image] pairs and combine them - idx = 0 - while idx < len(input): - current = input[idx] - next_elem = input[idx + 1] if idx + 1 < len(input) else None - if ( - isinstance(current, str) - and not is_base64_encoded(current) - and next_elem is not None - and is_base64_encoded(next_elem) - ): - # Text followed by image → combined request - transformed_request: ( - AmazonEmbeddingRequest - ) = AmazonTitanMultimodalEmbeddingG1Config()._transform_request( - input=next_elem, - inference_params=inference_params, - input_text=current, - ) - batch_data.append(transformed_request) - idx += 2 - else: + if combine_text_image_pairs: + # Scan for adjacent [text, data:image/...] pairs and combine them + idx = 0 + while idx < len(input): + current = input[idx] + next_elem = input[idx + 1] if idx + 1 < len(input) else None + if ( + isinstance(current, str) + and not current.startswith("data:") + and next_elem is not None + and isinstance(next_elem, str) + and next_elem.startswith("data:") + ): + # Text followed by image → combined request + transformed_request: ( + AmazonEmbeddingRequest + ) = AmazonTitanMultimodalEmbeddingG1Config()._transform_request( + input=next_elem, + inference_params=inference_params, + input_text=current, + ) + batch_data.append(transformed_request) + idx += 2 + else: + transformed_request = ( + AmazonTitanMultimodalEmbeddingG1Config()._transform_request( + input=current, inference_params=inference_params + ) + ) + batch_data.append(transformed_request) + idx += 1 + else: + # Default: process each element independently (backward-compatible) + for i in input: transformed_request = ( AmazonTitanMultimodalEmbeddingG1Config()._transform_request( - input=current, inference_params=inference_params + input=i, inference_params=inference_params ) ) batch_data.append(transformed_request) - idx += 1 else: for i in input: if model == "amazon.titan-embed-text-v1": @@ -516,39 +529,51 @@ class BedrockEmbedding(BaseAWSLLM): batch_data.append(twelvelabs_request) elif provider == "nova": batch_data = [] - # Scan for adjacent [text, data:image/...] pairs and combine them - idx = 0 - while idx < len(input): - current = input[idx] - next_elem = input[idx + 1] if idx + 1 < len(input) else None - if ( - isinstance(current, str) - and not current.startswith("data:") - and next_elem is not None - and isinstance(next_elem, str) - and next_elem.startswith("data:image/") - ): - # Text followed by image → combined request + if combine_text_image_pairs: + # Scan for adjacent [text, data:image/...] pairs and combine them + idx = 0 + while idx < len(input): + current = input[idx] + next_elem = input[idx + 1] if idx + 1 < len(input) else None + if ( + isinstance(current, str) + and not current.startswith("data:") + and next_elem is not None + and isinstance(next_elem, str) + and next_elem.startswith("data:image/") + ): + # Text followed by image → combined request + nova_request = AmazonNovaEmbeddingConfig()._transform_request( + input=next_elem, + inference_params=inference_params, + async_invoke_route=has_async_invoke, + model_id=modelId, + output_s3_uri=inference_params.get("output_s3_uri"), + input_text=current, + ) + batch_data.append(nova_request) + idx += 2 + else: + nova_request = AmazonNovaEmbeddingConfig()._transform_request( + input=current, + inference_params=inference_params, + async_invoke_route=has_async_invoke, + model_id=modelId, + output_s3_uri=inference_params.get("output_s3_uri"), + ) + batch_data.append(nova_request) + idx += 1 + else: + # Default: process each element independently (backward-compatible) + for i in input: nova_request = AmazonNovaEmbeddingConfig()._transform_request( - input=next_elem, - inference_params=inference_params, - async_invoke_route=has_async_invoke, - model_id=modelId, - output_s3_uri=inference_params.get("output_s3_uri"), - input_text=current, - ) - batch_data.append(nova_request) - idx += 2 - else: - nova_request = AmazonNovaEmbeddingConfig()._transform_request( - input=current, + input=i, inference_params=inference_params, async_invoke_route=has_async_invoke, model_id=modelId, output_s3_uri=inference_params.get("output_s3_uri"), ) batch_data.append(nova_request) - idx += 1 ### SET RUNTIME ENDPOINT ### endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( diff --git a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py index a332541d9a2..5d9c595adcd 100644 --- a/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py +++ b/tests/test_litellm/llms/bedrock/embed/test_bedrock_embedding.py @@ -1021,6 +1021,7 @@ def test_titan_multimodal_combined_text_and_image_request(): aws_access_key_id='fake', aws_secret_access_key='fake', aws_region_name='us-east-1', + combine_text_image_pairs=True, ) assert isinstance(response, litellm.EmbeddingResponse) @@ -1140,6 +1141,7 @@ def test_nova_combined_text_and_image_http_request(): aws_access_key_id='fake', aws_secret_access_key='fake', aws_region_name='us-east-1', + combine_text_image_pairs=True, ) assert isinstance(response, litellm.EmbeddingResponse) @@ -1151,3 +1153,145 @@ def test_nova_combined_text_and_image_http_request(): assert 'image' in params assert 'text' in params assert params['text']['value'] == 'shoes photo' + + +# ────────────────────────────────────────────────────────────────────────────── +# combine_text_image_pairs opt-in flag tests +# ────────────────────────────────────────────────────────────────────────────── + + +def test_titan_multimodal_no_combine_flag_returns_separate(): + """Without combine_text_image_pairs, [text, image] must produce 2 separate HTTP calls (backward compat).""" + client = HTTPHandler() + embed_response = { + 'embedding': [0.1, 0.2, 0.3], + 'inputTextTokenCount': 5, + } + + with patch.object(client, 'post') as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(embed_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + response = litellm.embedding( + model='bedrock/amazon.titan-embed-image-v1', + input=[ + 'Red leather handbag', + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==', + ], + client=client, + aws_access_key_id='fake', + aws_secret_access_key='fake', + aws_region_name='us-east-1', + # NOTE: combine_text_image_pairs NOT passed — default False + ) + + assert isinstance(response, litellm.EmbeddingResponse) + # Each element processed independently → 2 separate HTTP calls + assert mock_post.call_count == 2 + + +def test_titan_multimodal_combine_flag_returns_fused(): + """With combine_text_image_pairs=True, [text, image] must produce 1 fused HTTP call.""" + client = HTTPHandler() + embed_response = { + 'embedding': [0.1, 0.2, 0.3], + 'inputTextTokenCount': 5, + } + + with patch.object(client, 'post') as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(embed_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + response = litellm.embedding( + model='bedrock/amazon.titan-embed-image-v1', + input=[ + 'Red leather handbag', + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==', + ], + client=client, + aws_access_key_id='fake', + aws_secret_access_key='fake', + aws_region_name='us-east-1', + combine_text_image_pairs=True, + ) + + assert isinstance(response, litellm.EmbeddingResponse) + assert mock_post.call_count == 1 + + request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}')) + assert 'inputText' in request_body + assert 'inputImage' in request_body + + +def test_nova_text_only_standalone(): + """Nova with a single text input (no pairing) produces 1 call with only text in singleEmbeddingParams.""" + client = HTTPHandler() + nova_response = { + 'embeddings': [ + {'embeddingType': 'TEXT', 'embedding': [0.1, 0.2, 0.3]}, + ] + } + + with patch.object(client, 'post') as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(nova_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + response = litellm.embedding( + model='bedrock/amazon.nova-2-multimodal-embeddings-v1:0', + input=['just some text'], + client=client, + aws_access_key_id='fake', + aws_secret_access_key='fake', + aws_region_name='us-east-1', + ) + + assert isinstance(response, litellm.EmbeddingResponse) + assert mock_post.call_count == 1 + + request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}')) + params = request_body.get('singleEmbeddingParams', {}) + assert 'text' in params + assert 'image' not in params + + +def test_nova_image_only_standalone(): + """Nova with a single data:image/... input (no pairing) produces 1 call with only image in singleEmbeddingParams.""" + client = HTTPHandler() + nova_response = { + 'embeddings': [ + {'embeddingType': 'IMAGE', 'embedding': [0.1, 0.2, 0.3]}, + ] + } + + with patch.object(client, 'post') as mock_post: + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = json.dumps(nova_response) + mock_response.json = lambda: json.loads(mock_response.text) + mock_post.return_value = mock_response + + response = litellm.embedding( + model='bedrock/amazon.nova-2-multimodal-embeddings-v1:0', + input=['data:image/jpeg;base64,/9j/4AAQSkZJRgAB'], + client=client, + aws_access_key_id='fake', + aws_secret_access_key='fake', + aws_region_name='us-east-1', + ) + + assert isinstance(response, litellm.EmbeddingResponse) + assert mock_post.call_count == 1 + + request_body = json.loads(mock_post.call_args.kwargs.get('data', '{}')) + params = request_body.get('singleEmbeddingParams', {}) + assert 'image' in params + assert 'text' not in params