feat(gemini): support combined multimodal embeddings via nested input

Allows wrapping multiple inputs in a nested list to produce a single
combined embedding (text + image = 1 vector). Flat lists continue to
produce separate embeddings per input (OpenAI-compatible default).

Examples:
  input=["text", "image"]        → 2 separate embeddings
  input=[["text", "image"]]      → 1 combined embedding
  input=[["text", "image"], "x"] → 2 embeddings (1 combined + 1 separate)
This commit is contained in:
Chesars 2026-03-22 00:53:07 -03:00
parent 4694a30b62
commit ca37ced620
3 changed files with 124 additions and 23 deletions

View file

@ -566,6 +566,56 @@ curl -X POST http://localhost:4000/embeddings \
**Optional:** `dimensions` maps to Gemini's `outputDimensionality`.
#### Combined Multimodal Embeddings
By default, each element in the `input` list produces a **separate** embedding (OpenAI-compatible). To combine multiple inputs into a **single** embedding (e.g., text + image representing one entity), wrap them in a nested list:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import embedding
# Separate: 2 inputs → 2 embeddings
response = embedding(
model="gemini/gemini-embedding-2-preview",
input=["a red shoe", "data:image/png;base64,..."],
)
# response.data has 2 embeddings
# Combined: text + image → 1 embedding
response = embedding(
model="gemini/gemini-embedding-2-preview",
input=[["a red shoe", "data:image/png;base64,..."]],
)
# response.data has 1 embedding representing both together
# Mixed: 1 combined + 1 separate → 2 embeddings
response = embedding(
model="gemini/gemini-embedding-2-preview",
input=[["a red shoe", "data:image/png;base64,..."], "just text"],
)
# response.data has 2 embeddings
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl -X POST http://localhost:4000/embeddings \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gemini-embedding-2-preview",
"input": [["a red shoe", "data:image/png;base64,..."], "just text"]
}'
```
</TabItem>
</Tabs>
This is useful for representing multi-modal entities (e.g., a product with a name + photo) as a single vector for search and retrieval.
## Vertex AI Embedding Models

View file

@ -116,31 +116,38 @@ def _parse_data_url(data_url: str) -> Tuple[str, str]:
def _is_multimodal_input(input: EmbeddingInput) -> bool:
"""
Check if the input contains multimodal data (data URIs, file references, or GCS URLs).
Check if the input contains multimodal data (data URIs, file references,
GCS URLs, or nested lists for combined embeddings).
Args:
input: EmbeddingInput (str or List[str])
input: EmbeddingInput str, List[str], or List[Union[str, List[str]]]
Returns:
bool: True if any element is a data URI, file reference, or GCS URL
bool: True if any element is multimodal or a nested list
"""
if isinstance(input, str):
input_list = [input]
else:
input_list = input
return _is_multimodal_element(input)
for element in input_list:
if isinstance(element, str):
if element.startswith("data:") and ";base64," in element:
return True
if _is_file_reference(element):
return True
if _is_gcs_url(element):
return True
for element in input:
if isinstance(element, list):
return True
if isinstance(element, str) and _is_multimodal_element(element):
return True
return False
def _is_multimodal_element(element: str) -> bool:
"""Check if a single string element is multimodal."""
if element.startswith("data:") and ";base64," in element:
return True
if _is_file_reference(element):
return True
if _is_gcs_url(element):
return True
return False
def _build_part_for_input(
element: str,
resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
@ -186,6 +193,15 @@ def transform_openai_input_gemini_content(
Each input element becomes a separate EmbedContentRequest, supporting
text, data URIs, file references, and GCS URLs.
If an element is a list (nested input), all sub-elements are combined
into a single content with multiple parts, producing one combined
embedding for the group.
Examples:
input=["text", "image"] 2 separate embeddings
input=[["text", "image"]] 1 combined embedding
input=[["text", "image"], "x"] 2 embeddings (1 combined + 1 separate)
"""
gemini_model_name = "models/{}".format(model)
@ -199,10 +215,16 @@ def transform_openai_input_gemini_content(
requests: List[EmbedContentRequest] = []
for element in input_list:
part = _build_part_for_input(element, resolved_files=resolved_files)
if isinstance(element, list):
parts = [
_build_part_for_input(sub, resolved_files=resolved_files)
for sub in element
]
else:
parts = [_build_part_for_input(element, resolved_files=resolved_files)]
request = EmbedContentRequest(
model=gemini_model_name,
content=ContentType(parts=[part]),
content=ContentType(parts=parts),
**gemini_params,
)
requests.append(request)
@ -318,13 +340,15 @@ def process_response(
if _is_multimodal_input(input):
input_list = input if isinstance(input, list) else [input]
text_elements = [
e for e in input_list
if isinstance(e, str)
and not (e.startswith("data:") and ";base64," in e)
and not _is_gcs_url(e)
and not _is_file_reference(e)
]
text_elements = []
for e in input_list:
if isinstance(e, list):
text_elements.extend(
sub for sub in e
if isinstance(sub, str) and not _is_multimodal_element(sub)
)
elif isinstance(e, str) and not _is_multimodal_element(e):
text_elements.append(e)
if text_elements:
input_text = get_formatted_prompt(data={"input": text_elements}, call_type="embedding")
prompt_tokens = token_counter(model=model, text=input_text)

View file

@ -135,6 +135,33 @@ class TestTransformOpenaiInputGeminiContent:
)
assert len(result["requests"]) == 3
def test_nested_input_combined_embedding(self):
"""Nested list produces one request with multiple parts (combined embedding)."""
result = transform_openai_input_gemini_content(
input=[["a red shoe", IMAGE_DATA_URI]],
model="gemini-embedding-2-preview",
optional_params={},
)
assert len(result["requests"]) == 1
parts = result["requests"][0]["content"]["parts"]
assert len(parts) == 2
assert parts[0]["text"] == "a red shoe"
assert parts[1]["inline_data"] is not None
def test_mixed_nested_and_flat(self):
"""Mixed nested + flat produces correct number of requests."""
result = transform_openai_input_gemini_content(
input=[["text", IMAGE_DATA_URI], "standalone"],
model="gemini-embedding-2-preview",
optional_params={},
)
assert len(result["requests"]) == 2
# First: combined (2 parts)
assert len(result["requests"][0]["content"]["parts"]) == 2
# Second: standalone (1 part)
assert len(result["requests"][1]["content"]["parts"]) == 1
assert result["requests"][1]["content"]["parts"][0]["text"] == "standalone"
class TestTransformOpenaiInputGeminiEmbedContent:
"""Test transform_openai_input_gemini_embed_content (vertex_ai / embedContent path)."""