From ca37ced62024cb780aebf3936e46cf79ea81bec9 Mon Sep 17 00:00:00 2001 From: Chesars Date: Sun, 22 Mar 2026 00:53:07 -0300 Subject: [PATCH] feat(gemini): support combined multimodal embeddings via nested input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows wrapping multiple inputs in a nested list to produce a single combined embedding (text + image = 1 vector). Flat lists continue to produce separate embeddings per input (OpenAI-compatible default). Examples: input=["text", "image"] → 2 separate embeddings input=[["text", "image"]] → 1 combined embedding input=[["text", "image"], "x"] → 2 embeddings (1 combined + 1 separate) --- .../docs/embedding/supported_embedding.md | 50 +++++++++++++ .../batch_embed_content_transformation.py | 70 +++++++++++++------ ...test_batch_embed_content_transformation.py | 27 +++++++ 3 files changed, 124 insertions(+), 23 deletions(-) diff --git a/docs/my-website/docs/embedding/supported_embedding.md b/docs/my-website/docs/embedding/supported_embedding.md index 87acd0b33a5..700211e5bef 100644 --- a/docs/my-website/docs/embedding/supported_embedding.md +++ b/docs/my-website/docs/embedding/supported_embedding.md @@ -566,6 +566,56 @@ curl -X POST http://localhost:4000/embeddings \ **Optional:** `dimensions` maps to Gemini's `outputDimensionality`. +#### Combined Multimodal Embeddings + +By default, each element in the `input` list produces a **separate** embedding (OpenAI-compatible). To combine multiple inputs into a **single** embedding (e.g., text + image representing one entity), wrap them in a nested list: + + + + +```python +from litellm import embedding + +# Separate: 2 inputs → 2 embeddings +response = embedding( + model="gemini/gemini-embedding-2-preview", + input=["a red shoe", "data:image/png;base64,..."], +) +# response.data has 2 embeddings + +# Combined: text + image → 1 embedding +response = embedding( + model="gemini/gemini-embedding-2-preview", + input=[["a red shoe", "data:image/png;base64,..."]], +) +# response.data has 1 embedding representing both together + +# Mixed: 1 combined + 1 separate → 2 embeddings +response = embedding( + model="gemini/gemini-embedding-2-preview", + input=[["a red shoe", "data:image/png;base64,..."], "just text"], +) +# response.data has 2 embeddings +``` + + + + +```bash +curl -X POST http://localhost:4000/embeddings \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemini-embedding-2-preview", + "input": [["a red shoe", "data:image/png;base64,..."], "just text"] + }' +``` + + + + +This is useful for representing multi-modal entities (e.g., a product with a name + photo) as a single vector for search and retrieval. + ## Vertex AI Embedding Models diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 49135fc66f5..5777948c4f6 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -116,31 +116,38 @@ def _parse_data_url(data_url: str) -> Tuple[str, str]: def _is_multimodal_input(input: EmbeddingInput) -> bool: """ - Check if the input contains multimodal data (data URIs, file references, or GCS URLs). + Check if the input contains multimodal data (data URIs, file references, + GCS URLs, or nested lists for combined embeddings). Args: - input: EmbeddingInput (str or List[str]) + input: EmbeddingInput — str, List[str], or List[Union[str, List[str]]] Returns: - bool: True if any element is a data URI, file reference, or GCS URL + bool: True if any element is multimodal or a nested list """ if isinstance(input, str): - input_list = [input] - else: - input_list = input + return _is_multimodal_element(input) - for element in input_list: - if isinstance(element, str): - if element.startswith("data:") and ";base64," in element: - return True - if _is_file_reference(element): - return True - if _is_gcs_url(element): - return True + for element in input: + if isinstance(element, list): + return True + if isinstance(element, str) and _is_multimodal_element(element): + return True return False +def _is_multimodal_element(element: str) -> bool: + """Check if a single string element is multimodal.""" + if element.startswith("data:") and ";base64," in element: + return True + if _is_file_reference(element): + return True + if _is_gcs_url(element): + return True + return False + + def _build_part_for_input( element: str, resolved_files: Optional[Dict[str, Dict[str, str]]] = None, @@ -186,6 +193,15 @@ def transform_openai_input_gemini_content( Each input element becomes a separate EmbedContentRequest, supporting text, data URIs, file references, and GCS URLs. + + If an element is a list (nested input), all sub-elements are combined + into a single content with multiple parts, producing one combined + embedding for the group. + + Examples: + input=["text", "image"] → 2 separate embeddings + input=[["text", "image"]] → 1 combined embedding + input=[["text", "image"], "x"] → 2 embeddings (1 combined + 1 separate) """ gemini_model_name = "models/{}".format(model) @@ -199,10 +215,16 @@ def transform_openai_input_gemini_content( requests: List[EmbedContentRequest] = [] for element in input_list: - part = _build_part_for_input(element, resolved_files=resolved_files) + if isinstance(element, list): + parts = [ + _build_part_for_input(sub, resolved_files=resolved_files) + for sub in element + ] + else: + parts = [_build_part_for_input(element, resolved_files=resolved_files)] request = EmbedContentRequest( model=gemini_model_name, - content=ContentType(parts=[part]), + content=ContentType(parts=parts), **gemini_params, ) requests.append(request) @@ -318,13 +340,15 @@ def process_response( if _is_multimodal_input(input): input_list = input if isinstance(input, list) else [input] - text_elements = [ - e for e in input_list - if isinstance(e, str) - and not (e.startswith("data:") and ";base64," in e) - and not _is_gcs_url(e) - and not _is_file_reference(e) - ] + text_elements = [] + for e in input_list: + if isinstance(e, list): + text_elements.extend( + sub for sub in e + if isinstance(sub, str) and not _is_multimodal_element(sub) + ) + elif isinstance(e, str) and not _is_multimodal_element(e): + text_elements.append(e) if text_elements: input_text = get_formatted_prompt(data={"input": text_elements}, call_type="embedding") prompt_tokens = token_counter(model=model, text=input_text) diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 1b295994b50..98b2c0167af 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -135,6 +135,33 @@ class TestTransformOpenaiInputGeminiContent: ) assert len(result["requests"]) == 3 + def test_nested_input_combined_embedding(self): + """Nested list produces one request with multiple parts (combined embedding).""" + result = transform_openai_input_gemini_content( + input=[["a red shoe", IMAGE_DATA_URI]], + model="gemini-embedding-2-preview", + optional_params={}, + ) + assert len(result["requests"]) == 1 + parts = result["requests"][0]["content"]["parts"] + assert len(parts) == 2 + assert parts[0]["text"] == "a red shoe" + assert parts[1]["inline_data"] is not None + + def test_mixed_nested_and_flat(self): + """Mixed nested + flat produces correct number of requests.""" + result = transform_openai_input_gemini_content( + input=[["text", IMAGE_DATA_URI], "standalone"], + model="gemini-embedding-2-preview", + optional_params={}, + ) + assert len(result["requests"]) == 2 + # First: combined (2 parts) + assert len(result["requests"][0]["content"]["parts"]) == 2 + # Second: standalone (1 part) + assert len(result["requests"][1]["content"]["parts"]) == 1 + assert result["requests"][1]["content"]["parts"][0]["text"] == "standalone" + class TestTransformOpenaiInputGeminiEmbedContent: """Test transform_openai_input_gemini_embed_content (vertex_ai / embedContent path)."""