mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-17 23:51:30 +00:00
feat(gemini): support combined multimodal embeddings via nested input
Allows wrapping multiple inputs in a nested list to produce a single combined embedding (text + image = 1 vector). Flat lists continue to produce separate embeddings per input (OpenAI-compatible default). Examples: input=["text", "image"] → 2 separate embeddings input=[["text", "image"]] → 1 combined embedding input=[["text", "image"], "x"] → 2 embeddings (1 combined + 1 separate)
This commit is contained in:
parent
4694a30b62
commit
ca37ced620
3 changed files with 124 additions and 23 deletions
|
|
@ -566,6 +566,56 @@ curl -X POST http://localhost:4000/embeddings \
|
|||
|
||||
**Optional:** `dimensions` maps to Gemini's `outputDimensionality`.
|
||||
|
||||
#### Combined Multimodal Embeddings
|
||||
|
||||
By default, each element in the `input` list produces a **separate** embedding (OpenAI-compatible). To combine multiple inputs into a **single** embedding (e.g., text + image representing one entity), wrap them in a nested list:
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import embedding
|
||||
|
||||
# Separate: 2 inputs → 2 embeddings
|
||||
response = embedding(
|
||||
model="gemini/gemini-embedding-2-preview",
|
||||
input=["a red shoe", "data:image/png;base64,..."],
|
||||
)
|
||||
# response.data has 2 embeddings
|
||||
|
||||
# Combined: text + image → 1 embedding
|
||||
response = embedding(
|
||||
model="gemini/gemini-embedding-2-preview",
|
||||
input=[["a red shoe", "data:image/png;base64,..."]],
|
||||
)
|
||||
# response.data has 1 embedding representing both together
|
||||
|
||||
# Mixed: 1 combined + 1 separate → 2 embeddings
|
||||
response = embedding(
|
||||
model="gemini/gemini-embedding-2-preview",
|
||||
input=[["a red shoe", "data:image/png;base64,..."], "just text"],
|
||||
)
|
||||
# response.data has 2 embeddings
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:4000/embeddings \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "gemini-embedding-2-preview",
|
||||
"input": [["a red shoe", "data:image/png;base64,..."], "just text"]
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
This is useful for representing multi-modal entities (e.g., a product with a name + photo) as a single vector for search and retrieval.
|
||||
|
||||
|
||||
## Vertex AI Embedding Models
|
||||
|
||||
|
|
|
|||
|
|
@ -116,31 +116,38 @@ def _parse_data_url(data_url: str) -> Tuple[str, str]:
|
|||
|
||||
def _is_multimodal_input(input: EmbeddingInput) -> bool:
|
||||
"""
|
||||
Check if the input contains multimodal data (data URIs, file references, or GCS URLs).
|
||||
Check if the input contains multimodal data (data URIs, file references,
|
||||
GCS URLs, or nested lists for combined embeddings).
|
||||
|
||||
Args:
|
||||
input: EmbeddingInput (str or List[str])
|
||||
input: EmbeddingInput — str, List[str], or List[Union[str, List[str]]]
|
||||
|
||||
Returns:
|
||||
bool: True if any element is a data URI, file reference, or GCS URL
|
||||
bool: True if any element is multimodal or a nested list
|
||||
"""
|
||||
if isinstance(input, str):
|
||||
input_list = [input]
|
||||
else:
|
||||
input_list = input
|
||||
return _is_multimodal_element(input)
|
||||
|
||||
for element in input_list:
|
||||
if isinstance(element, str):
|
||||
if element.startswith("data:") and ";base64," in element:
|
||||
return True
|
||||
if _is_file_reference(element):
|
||||
return True
|
||||
if _is_gcs_url(element):
|
||||
return True
|
||||
for element in input:
|
||||
if isinstance(element, list):
|
||||
return True
|
||||
if isinstance(element, str) and _is_multimodal_element(element):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _is_multimodal_element(element: str) -> bool:
|
||||
"""Check if a single string element is multimodal."""
|
||||
if element.startswith("data:") and ";base64," in element:
|
||||
return True
|
||||
if _is_file_reference(element):
|
||||
return True
|
||||
if _is_gcs_url(element):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _build_part_for_input(
|
||||
element: str,
|
||||
resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
|
||||
|
|
@ -186,6 +193,15 @@ def transform_openai_input_gemini_content(
|
|||
|
||||
Each input element becomes a separate EmbedContentRequest, supporting
|
||||
text, data URIs, file references, and GCS URLs.
|
||||
|
||||
If an element is a list (nested input), all sub-elements are combined
|
||||
into a single content with multiple parts, producing one combined
|
||||
embedding for the group.
|
||||
|
||||
Examples:
|
||||
input=["text", "image"] → 2 separate embeddings
|
||||
input=[["text", "image"]] → 1 combined embedding
|
||||
input=[["text", "image"], "x"] → 2 embeddings (1 combined + 1 separate)
|
||||
"""
|
||||
gemini_model_name = "models/{}".format(model)
|
||||
|
||||
|
|
@ -199,10 +215,16 @@ def transform_openai_input_gemini_content(
|
|||
requests: List[EmbedContentRequest] = []
|
||||
|
||||
for element in input_list:
|
||||
part = _build_part_for_input(element, resolved_files=resolved_files)
|
||||
if isinstance(element, list):
|
||||
parts = [
|
||||
_build_part_for_input(sub, resolved_files=resolved_files)
|
||||
for sub in element
|
||||
]
|
||||
else:
|
||||
parts = [_build_part_for_input(element, resolved_files=resolved_files)]
|
||||
request = EmbedContentRequest(
|
||||
model=gemini_model_name,
|
||||
content=ContentType(parts=[part]),
|
||||
content=ContentType(parts=parts),
|
||||
**gemini_params,
|
||||
)
|
||||
requests.append(request)
|
||||
|
|
@ -318,13 +340,15 @@ def process_response(
|
|||
|
||||
if _is_multimodal_input(input):
|
||||
input_list = input if isinstance(input, list) else [input]
|
||||
text_elements = [
|
||||
e for e in input_list
|
||||
if isinstance(e, str)
|
||||
and not (e.startswith("data:") and ";base64," in e)
|
||||
and not _is_gcs_url(e)
|
||||
and not _is_file_reference(e)
|
||||
]
|
||||
text_elements = []
|
||||
for e in input_list:
|
||||
if isinstance(e, list):
|
||||
text_elements.extend(
|
||||
sub for sub in e
|
||||
if isinstance(sub, str) and not _is_multimodal_element(sub)
|
||||
)
|
||||
elif isinstance(e, str) and not _is_multimodal_element(e):
|
||||
text_elements.append(e)
|
||||
if text_elements:
|
||||
input_text = get_formatted_prompt(data={"input": text_elements}, call_type="embedding")
|
||||
prompt_tokens = token_counter(model=model, text=input_text)
|
||||
|
|
|
|||
|
|
@ -135,6 +135,33 @@ class TestTransformOpenaiInputGeminiContent:
|
|||
)
|
||||
assert len(result["requests"]) == 3
|
||||
|
||||
def test_nested_input_combined_embedding(self):
|
||||
"""Nested list produces one request with multiple parts (combined embedding)."""
|
||||
result = transform_openai_input_gemini_content(
|
||||
input=[["a red shoe", IMAGE_DATA_URI]],
|
||||
model="gemini-embedding-2-preview",
|
||||
optional_params={},
|
||||
)
|
||||
assert len(result["requests"]) == 1
|
||||
parts = result["requests"][0]["content"]["parts"]
|
||||
assert len(parts) == 2
|
||||
assert parts[0]["text"] == "a red shoe"
|
||||
assert parts[1]["inline_data"] is not None
|
||||
|
||||
def test_mixed_nested_and_flat(self):
|
||||
"""Mixed nested + flat produces correct number of requests."""
|
||||
result = transform_openai_input_gemini_content(
|
||||
input=[["text", IMAGE_DATA_URI], "standalone"],
|
||||
model="gemini-embedding-2-preview",
|
||||
optional_params={},
|
||||
)
|
||||
assert len(result["requests"]) == 2
|
||||
# First: combined (2 parts)
|
||||
assert len(result["requests"][0]["content"]["parts"]) == 2
|
||||
# Second: standalone (1 part)
|
||||
assert len(result["requests"][1]["content"]["parts"]) == 1
|
||||
assert result["requests"][1]["content"]["parts"][0]["text"] == "standalone"
|
||||
|
||||
|
||||
class TestTransformOpenaiInputGeminiEmbedContent:
|
||||
"""Test transform_openai_input_gemini_embed_content (vertex_ai / embedContent path)."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue