From ca37ced62024cb780aebf3936e46cf79ea81bec9 Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 22 Mar 2026 00:53:07 -0300 Subject: [PATCH 1/3] feat(gemini): support combined multimodal embeddings via nested input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows wrapping multiple inputs in a nested list to produce a single combined embedding (text + image = 1 vector). Flat lists continue to produce separate embeddings per input (OpenAI-compatible default). Examples: input=["text", "image"] → 2 separate embeddings input=[["text", "image"]] → 1 combined embedding input=[["text", "image"], "x"] → 2 embeddings (1 combined + 1 separate) --- .../docs/embedding/supported_embedding.md | 50 +++++++++++++ .../batch_embed_content_transformation.py | 70 +++++++++++++------ ...test_batch_embed_content_transformation.py | 27 +++++++ 3 files changed, 124 insertions(+), 23 deletions(-) diff --git a/docs/my-website/docs/embedding/supported_embedding.md b/docs/my-website/docs/embedding/supported_embedding.md index 87acd0b33a5..700211e5bef 100644 --- a/docs/my-website/docs/embedding/supported_embedding.md +++ b/docs/my-website/docs/embedding/supported_embedding.md @@ -566,6 +566,56 @@ curl -X POST http://localhost:4000/embeddings \ **Optional:** `dimensions` maps to Gemini's `outputDimensionality`. +#### Combined Multimodal Embeddings + +By default, each element in the `input` list produces a **separate** embedding (OpenAI-compatible). To combine multiple inputs into a **single** embedding (e.g., text + image representing one entity), wrap them in a nested list: + + + + +```python +from litellm import embedding + +# Separate: 2 inputs → 2 embeddings +response = embedding( + model="gemini/gemini-embedding-2-preview", + input=["a red shoe", "data:image/png;base64,..."], +) +# response.data has 2 embeddings + +# Combined: text + image → 1 embedding +response = embedding( + model="gemini/gemini-embedding-2-preview", + input=[["a red shoe", "data:image/png;base64,..."]], +) +# response.data has 1 embedding representing both together + +# Mixed: 1 combined + 1 separate → 2 embeddings +response = embedding( + model="gemini/gemini-embedding-2-preview", + input=[["a red shoe", "data:image/png;base64,..."], "just text"], +) +# response.data has 2 embeddings +``` + + + + +```bash +curl -X POST http://localhost:4000/embeddings \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemini-embedding-2-preview", + "input": [["a red shoe", "data:image/png;base64,..."], "just text"] + }' +``` + + + + +This is useful for representing multi-modal entities (e.g., a product with a name + photo) as a single vector for search and retrieval. + ## Vertex AI Embedding Models diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 49135fc66f5..5777948c4f6 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -116,31 +116,38 @@ def _parse_data_url(data_url: str) -> Tuple[str, str]: def _is_multimodal_input(input: EmbeddingInput) -> bool: """ - Check if the input contains multimodal data (data URIs, file references, or GCS URLs). + Check if the input contains multimodal data (data URIs, file references, + GCS URLs, or nested lists for combined embeddings). Args: - input: EmbeddingInput (str or List[str]) + input: EmbeddingInput — str, List[str], or List[Union[str, List[str]]] Returns: - bool: True if any element is a data URI, file reference, or GCS URL + bool: True if any element is multimodal or a nested list """ if isinstance(input, str): - input_list = [input] - else: - input_list = input + return _is_multimodal_element(input) - for element in input_list: - if isinstance(element, str): - if element.startswith("data:") and ";base64," in element: - return True - if _is_file_reference(element): - return True - if _is_gcs_url(element): - return True + for element in input: + if isinstance(element, list): + return True + if isinstance(element, str) and _is_multimodal_element(element): + return True return False +def _is_multimodal_element(element: str) -> bool: + """Check if a single string element is multimodal.""" + if element.startswith("data:") and ";base64," in element: + return True + if _is_file_reference(element): + return True + if _is_gcs_url(element): + return True + return False + + def _build_part_for_input( element: str, resolved_files: Optional[Dict[str, Dict[str, str]]] = None, @@ -186,6 +193,15 @@ def transform_openai_input_gemini_content( Each input element becomes a separate EmbedContentRequest, supporting text, data URIs, file references, and GCS URLs. + + If an element is a list (nested input), all sub-elements are combined + into a single content with multiple parts, producing one combined + embedding for the group. + + Examples: + input=["text", "image"] → 2 separate embeddings + input=[["text", "image"]] → 1 combined embedding + input=[["text", "image"], "x"] → 2 embeddings (1 combined + 1 separate) """ gemini_model_name = "models/{}".format(model) @@ -199,10 +215,16 @@ def transform_openai_input_gemini_content( requests: List[EmbedContentRequest] = [] for element in input_list: - part = _build_part_for_input(element, resolved_files=resolved_files) + if isinstance(element, list): + parts = [ + _build_part_for_input(sub, resolved_files=resolved_files) + for sub in element + ] + else: + parts = [_build_part_for_input(element, resolved_files=resolved_files)] request = EmbedContentRequest( model=gemini_model_name, - content=ContentType(parts=[part]), + content=ContentType(parts=parts), **gemini_params, ) requests.append(request) @@ -318,13 +340,15 @@ def process_response( if _is_multimodal_input(input): input_list = input if isinstance(input, list) else [input] - text_elements = [ - e for e in input_list - if isinstance(e, str) - and not (e.startswith("data:") and ";base64," in e) - and not _is_gcs_url(e) - and not _is_file_reference(e) - ] + text_elements = [] + for e in input_list: + if isinstance(e, list): + text_elements.extend( + sub for sub in e + if isinstance(sub, str) and not _is_multimodal_element(sub) + ) + elif isinstance(e, str) and not _is_multimodal_element(e): + text_elements.append(e) if text_elements: input_text = get_formatted_prompt(data={"input": text_elements}, call_type="embedding") prompt_tokens = token_counter(model=model, text=input_text) diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 1b295994b50..98b2c0167af 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -135,6 +135,33 @@ class TestTransformOpenaiInputGeminiContent: ) assert len(result["requests"]) == 3 + def test_nested_input_combined_embedding(self): + """Nested list produces one request with multiple parts (combined embedding).""" + result = transform_openai_input_gemini_content( + input=[["a red shoe", IMAGE_DATA_URI]], + model="gemini-embedding-2-preview", + optional_params={}, + ) + assert len(result["requests"]) == 1 + parts = result["requests"][0]["content"]["parts"] + assert len(parts) == 2 + assert parts[0]["text"] == "a red shoe" + assert parts[1]["inline_data"] is not None + + def test_mixed_nested_and_flat(self): + """Mixed nested + flat produces correct number of requests.""" + result = transform_openai_input_gemini_content( + input=[["text", IMAGE_DATA_URI], "standalone"], + model="gemini-embedding-2-preview", + optional_params={}, + ) + assert len(result["requests"]) == 2 + # First: combined (2 parts) + assert len(result["requests"][0]["content"]["parts"]) == 2 + # Second: standalone (1 part) + assert len(result["requests"][1]["content"]["parts"]) == 1 + assert result["requests"][1]["content"]["parts"][0]["text"] == "standalone" + class TestTransformOpenaiInputGeminiEmbedContent: """Test transform_openai_input_gemini_embed_content (vertex_ai / embedContent path).""" From 960108939d7bbebfb9cf56d485484cbc7c3a260d Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 22 Mar 2026 01:01:00 -0300 Subject: [PATCH 2/3] fix: update EmbeddingInput type, validate nested sub-elements, add tests --- .../batch_embed_content_transformation.py | 7 +++++++ litellm/types/llms/openai.py | 2 +- .../test_batch_embed_content_transformation.py | 6 ++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 5777948c4f6..66ff9ef987c 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -216,6 +216,13 @@ def transform_openai_input_gemini_content( for element in input_list: if isinstance(element, list): + if not element: + raise ValueError("Nested input list must not be empty") + for sub in element: + if not isinstance(sub, str): + raise ValueError( + f"Elements inside a nested input list must be strings, got {type(sub)}" + ) parts = [ _build_part_for_input(sub, resolved_files=resolved_files) for sub in element diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 5a80b40d61f..79bfe73d12e 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -103,7 +103,7 @@ FileTypes = Union[ ] -EmbeddingInput = Union[str, List[str]] +EmbeddingInput = Union[str, List[Union[str, List[str]]]] class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 98b2c0167af..7e757e32e0e 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -44,6 +44,12 @@ class TestIsMultimodalInput: def test_mixed_text_and_image(self): assert _is_multimodal_input(["hello", IMAGE_DATA_URI]) is True + def test_nested_list_is_multimodal(self): + assert _is_multimodal_input([["text_a", "text_b"]]) is True + + def test_nested_list_with_image_is_multimodal(self): + assert _is_multimodal_input([["a red shoe", IMAGE_DATA_URI]]) is True + class TestBuildPartForInput: def test_text_input(self): From 9fc6fdad667bb3e879b4995b082933c518e41049 Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 22 Mar 2026 01:08:01 -0300 Subject: [PATCH 3/3] fix: clear error for nested lists on embedContent path, add validation tests --- .../batch_embed_content_transformation.py | 5 ++++ ...test_batch_embed_content_transformation.py | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 66ff9ef987c..60b6a1a33ef 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -269,6 +269,11 @@ def transform_openai_input_gemini_embed_content( parts: List[PartType] = [] for element in input_list: + if isinstance(element, list): + raise ValueError( + "Nested (combined) embeddings are not supported on the embedContent path. " + "Use the batchEmbedContents path or pass a flat list instead." + ) if not isinstance(element, str): raise ValueError(f"Unsupported input type: {type(element)}") parts.append(_build_part_for_input(element, resolved_files=resolved_files)) diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 7e757e32e0e..9ba6e04744a 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -257,3 +257,33 @@ class TestProcessResponse: assert result.data[1]["index"] == 1 # Should count tokens only for the text element, not the image assert result.usage.prompt_tokens > 0 + + def test_nested_input_token_counting(self): + """Nested list: only plain-text sub-elements should be counted.""" + predictions: VertexAIBatchEmbeddingsResponseObject = { + "embeddings": [{"values": [0.1, 0.2]}] + } + result = process_response( + input=[["a red shoe", IMAGE_DATA_URI]], + model_response=EmbeddingResponse(), + model="gemini-embedding-2-preview", + _predictions=predictions, + ) + assert len(result.data) == 1 + assert result.usage.prompt_tokens > 0 + + def test_nested_empty_list_raises(self): + with pytest.raises(ValueError, match="must not be empty"): + transform_openai_input_gemini_content( + input=[[]], + model="gemini-embedding-2-preview", + optional_params={}, + ) + + def test_nested_non_string_element_raises(self): + with pytest.raises(ValueError, match="must be strings"): + transform_openai_input_gemini_content( + input=[[["doubly", "nested"]]], + model="gemini-embedding-2-preview", + optional_params={}, + )