Merge pull request #24341 from Chesars/feat/gemini-combined-multimodal-embeddings

feat(gemini): support combined multimodal embeddings via nested input

docs/my-website/docs/embedding/supported_embedding.md

@@ -566,6 +566,56 @@ curl -X POST http://localhost:4000/embeddings \
 
 **Optional:** `dimensions` maps to Gemini's `outputDimensionality`.
 
+#### Combined Multimodal Embeddings
+
+By default, each element in the `input` list produces a **separate** embedding (OpenAI-compatible). To combine multiple inputs into a **single** embedding (e.g., text + image representing one entity), wrap them in a nested list:
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import embedding
+
+# Separate: 2 inputs → 2 embeddings
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=["a red shoe", "data:image/png;base64,..."],
+)
+# response.data has 2 embeddings
+
+# Combined: text + image → 1 embedding
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=[["a red shoe", "data:image/png;base64,..."]],
+)
+# response.data has 1 embedding representing both together
+
+# Mixed: 1 combined + 1 separate → 2 embeddings
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=[["a red shoe", "data:image/png;base64,..."], "just text"],
+)
+# response.data has 2 embeddings
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash
+curl -X POST http://localhost:4000/embeddings \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gemini-embedding-2-preview",
+    "input": [["a red shoe", "data:image/png;base64,..."], "just text"]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+This is useful for representing multi-modal entities (e.g., a product with a name + photo) as a single vector for search and retrieval.
+
 
 ## Vertex AI Embedding Models
 

litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py

@@ -116,31 +116,38 @@ def _parse_data_url(data_url: str) -> Tuple[str, str]:
 
 def _is_multimodal_input(input: EmbeddingInput) -> bool:
     """
-    Check if the input contains multimodal data (data URIs, file references, or GCS URLs).
+    Check if the input contains multimodal data (data URIs, file references,
+    GCS URLs, or nested lists for combined embeddings).
 
     Args:
-        input: EmbeddingInput (str or List[str])
+        input: EmbeddingInput — str, List[str], or List[Union[str, List[str]]]
 
     Returns:
-        bool: True if any element is a data URI, file reference, or GCS URL
+        bool: True if any element is multimodal or a nested list
     """
     if isinstance(input, str):
-        input_list = [input]
-    else:
-        input_list = input
+        return _is_multimodal_element(input)
 
-    for element in input_list:
-        if isinstance(element, str):
-            if element.startswith("data:") and ";base64," in element:
-                return True
-            if _is_file_reference(element):
-                return True
-            if _is_gcs_url(element):
-                return True
+    for element in input:
+        if isinstance(element, list):
+            return True
+        if isinstance(element, str) and _is_multimodal_element(element):
+            return True
 
     return False
 
 
+def _is_multimodal_element(element: str) -> bool:
+    """Check if a single string element is multimodal."""
+    if element.startswith("data:") and ";base64," in element:
+        return True
+    if _is_file_reference(element):
+        return True
+    if _is_gcs_url(element):
+        return True
+    return False
+
+
 def _build_part_for_input(
     element: str,
     resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
@@ -186,6 +193,15 @@ def transform_openai_input_gemini_content(
 
     Each input element becomes a separate EmbedContentRequest, supporting
     text, data URIs, file references, and GCS URLs.
+
+    If an element is a list (nested input), all sub-elements are combined
+    into a single content with multiple parts, producing one combined
+    embedding for the group.
+
+    Examples:
+        input=["text", "image"]         → 2 separate embeddings
+        input=[["text", "image"]]       → 1 combined embedding
+        input=[["text", "image"], "x"]  → 2 embeddings (1 combined + 1 separate)
     """
     gemini_model_name = "models/{}".format(model)
 
@@ -199,10 +215,23 @@ def transform_openai_input_gemini_content(
     requests: List[EmbedContentRequest] = []
 
     for element in input_list:
-        part = _build_part_for_input(element, resolved_files=resolved_files)
+        if isinstance(element, list):
+            if not element:
+                raise ValueError("Nested input list must not be empty")
+            for sub in element:
+                if not isinstance(sub, str):
+                    raise ValueError(
+                        f"Elements inside a nested input list must be strings, got {type(sub)}"
+                    )
+            parts = [
+                _build_part_for_input(sub, resolved_files=resolved_files)
+                for sub in element
+            ]
+        else:
+            parts = [_build_part_for_input(element, resolved_files=resolved_files)]
         request = EmbedContentRequest(
             model=gemini_model_name,
-            content=ContentType(parts=[part]),
+            content=ContentType(parts=parts),
             **gemini_params,
         )
         requests.append(request)
@@ -240,6 +269,11 @@ def transform_openai_input_gemini_embed_content(
     parts: List[PartType] = []
 
     for element in input_list:
+        if isinstance(element, list):
+            raise ValueError(
+                "Nested (combined) embeddings are not supported on the embedContent path. "
+                "Use the batchEmbedContents path or pass a flat list instead."
+            )
         if not isinstance(element, str):
             raise ValueError(f"Unsupported input type: {type(element)}")
         parts.append(_build_part_for_input(element, resolved_files=resolved_files))
@@ -318,13 +352,15 @@ def process_response(
 
     if _is_multimodal_input(input):
         input_list = input if isinstance(input, list) else [input]
-        text_elements = [
-            e for e in input_list
-            if isinstance(e, str)
-            and not (e.startswith("data:") and ";base64," in e)
-            and not _is_gcs_url(e)
-            and not _is_file_reference(e)
-        ]
+        text_elements = []
+        for e in input_list:
+            if isinstance(e, list):
+                text_elements.extend(
+                    sub for sub in e
+                    if isinstance(sub, str) and not _is_multimodal_element(sub)
+                )
+            elif isinstance(e, str) and not _is_multimodal_element(e):
+                text_elements.append(e)
         if text_elements:
             input_text = get_formatted_prompt(data={"input": text_elements}, call_type="embedding")
             prompt_tokens = token_counter(model=model, text=input_text)

litellm/types/llms/openai.py

@@ -103,7 +103,7 @@ FileTypes = Union[
 ]
 
 
-EmbeddingInput = Union[str, List[str]]
+EmbeddingInput = Union[str, List[Union[str, List[str]]]]
 
 
 class HttpxBinaryResponseContent(_HttpxBinaryResponseContent):

tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py

@@ -44,6 +44,12 @@ class TestIsMultimodalInput:
     def test_mixed_text_and_image(self):
         assert _is_multimodal_input(["hello", IMAGE_DATA_URI]) is True
 
+    def test_nested_list_is_multimodal(self):
+        assert _is_multimodal_input([["text_a", "text_b"]]) is True
+
+    def test_nested_list_with_image_is_multimodal(self):
+        assert _is_multimodal_input([["a red shoe", IMAGE_DATA_URI]]) is True
+
 
 class TestBuildPartForInput:
     def test_text_input(self):
@@ -135,6 +141,33 @@ class TestTransformOpenaiInputGeminiContent:
         )
         assert len(result["requests"]) == 3
 
+    def test_nested_input_combined_embedding(self):
+        """Nested list produces one request with multiple parts (combined embedding)."""
+        result = transform_openai_input_gemini_content(
+            input=[["a red shoe", IMAGE_DATA_URI]],
+            model="gemini-embedding-2-preview",
+            optional_params={},
+        )
+        assert len(result["requests"]) == 1
+        parts = result["requests"][0]["content"]["parts"]
+        assert len(parts) == 2
+        assert parts[0]["text"] == "a red shoe"
+        assert parts[1]["inline_data"] is not None
+
+    def test_mixed_nested_and_flat(self):
+        """Mixed nested + flat produces correct number of requests."""
+        result = transform_openai_input_gemini_content(
+            input=[["text", IMAGE_DATA_URI], "standalone"],
+            model="gemini-embedding-2-preview",
+            optional_params={},
+        )
+        assert len(result["requests"]) == 2
+        # First: combined (2 parts)
+        assert len(result["requests"][0]["content"]["parts"]) == 2
+        # Second: standalone (1 part)
+        assert len(result["requests"][1]["content"]["parts"]) == 1
+        assert result["requests"][1]["content"]["parts"][0]["text"] == "standalone"
+
 
 class TestTransformOpenaiInputGeminiEmbedContent:
     """Test transform_openai_input_gemini_embed_content (vertex_ai / embedContent path)."""
@@ -224,3 +257,33 @@ class TestProcessResponse:
         assert result.data[1]["index"] == 1
         # Should count tokens only for the text element, not the image
         assert result.usage.prompt_tokens > 0
+
+    def test_nested_input_token_counting(self):
+        """Nested list: only plain-text sub-elements should be counted."""
+        predictions: VertexAIBatchEmbeddingsResponseObject = {
+            "embeddings": [{"values": [0.1, 0.2]}]
+        }
+        result = process_response(
+            input=[["a red shoe", IMAGE_DATA_URI]],
+            model_response=EmbeddingResponse(),
+            model="gemini-embedding-2-preview",
+            _predictions=predictions,
+        )
+        assert len(result.data) == 1
+        assert result.usage.prompt_tokens > 0
+
+    def test_nested_empty_list_raises(self):
+        with pytest.raises(ValueError, match="must not be empty"):
+            transform_openai_input_gemini_content(
+                input=[[]],
+                model="gemini-embedding-2-preview",
+                optional_params={},
+            )
+
+    def test_nested_non_string_element_raises(self):
+        with pytest.raises(ValueError, match="must be strings"):
+            transform_openai_input_gemini_content(
+                input=[[["doubly", "nested"]]],
+                model="gemini-embedding-2-preview",
+                optional_params={},
+            )