This commit is contained in:
ishaan-j-berri 2026-10-27 02:24:21 +00:00 • committed by GitHub
commit 0fc07c1ac9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 602 additions and 56 deletions

View file

@ -39,7 +39,7 @@ from litellm.litellm_core_utils.logging_utils import (
from litellm.types.caching import CACHED_STREAM_EVENTS_KEY, EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding
from litellm.types.integrations.custom_logger import converted_stream_requested
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.openai import ChatCompletionFileObject, ResponsesAPIResponse
from litellm.types.llms.openai import ChatCompletionFileObject, ChatCompletionTextObject, ResponsesAPIResponse
from litellm.types.rerank import RerankResponse
from litellm.types.utils import (
CachingDetails,
@ -71,7 +71,7 @@ from litellm.litellm_core_utils.core_helpers import (
)
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
EmbeddingCacheInputElement = str | list[int] | ChatCompletionFileObject
EmbeddingCacheInputElement = str | list[int] | ChatCompletionFileObject | ChatCompletionTextObject
class CachingHandlerResponse(LiteLLMBaseModel):

View file

@ -45,9 +45,10 @@ class GoogleBatchEmbeddings(VertexLLM):
@staticmethod
def _flatten_and_detect_file_refs(
input: GeminiEmbeddingInput,
drop_params: bool = False,
) -> tuple[list[str], bool]:
"""Flatten nested input lists and file content blocks into their sources and detect file references."""
flat_elements: Final = list(flatten_media_sources(input))
flat_elements: Final = list(flatten_media_sources(input, drop_params=drop_params))
has_file_refs: Final = any(is_file_reference(e) for e in flat_elements)
return flat_elements, has_file_refs
@ -218,7 +219,7 @@ class GoogleBatchEmbeddings(VertexLLM):
### TRANSFORMATION (sync path) ###
request_data: VertexAIBatchEmbeddingsRequestBody | dict[str, object]
flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input)
flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input, drop_params=drop_params)
if use_embed_content:
resolved_files = {}
if api_key:
@ -324,7 +325,7 @@ class GoogleBatchEmbeddings(VertexLLM):
async_handler = client
### TRANSFORMATION (async path) ###
flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input)
flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input, drop_params=drop_params)
if use_embed_content:
resolved_files = {}
if api_key:

View file

@ -166,11 +166,20 @@ class _EmbeddingFileBlock(LiteLLMBaseModel):
file: _EmbeddingFile
class _EmbeddingTextBlock(LiteLLMBaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
type: Literal["text"]
text: str
_file_block_adapter: Final = TypeAdapter(_EmbeddingFileBlock)
_text_block_adapter: Final = TypeAdapter(_EmbeddingTextBlock)
_video_metadata_adapter: Final = TypeAdapter(VideoMetadataType)
_input_shape_adapter: Final[TypeAdapter[str | list[object]]] = TypeAdapter(str | list[object])
_mapping_adapter: Final = TypeAdapter(dict[str, object])
_BLOCK_FIELDS: Final = frozenset(_EmbeddingFileBlock.model_fields)
_TEXT_BLOCK_FIELDS: Final = frozenset(_EmbeddingTextBlock.model_fields)
_FILE_FIELDS: Final = frozenset(_EmbeddingFile.model_fields)
_VIDEO_METADATA_FIELDS: Final = frozenset(_EmbeddingVideoMetadata.model_fields)
_FILE_SOURCE_FORMS: Final = "a data: URI, a gs:// URL, a files/ reference, or a Gemini Files API URI"
@ -211,10 +220,31 @@ def _dropping_unsupported_keys(block: Mapping[str, object]) -> dict[str, object]
}
def _is_text_block(element: object) -> bool:
return (
isinstance(element, Mapping)
and cast(Mapping[str, object], element).get("type") == "text" # cast-ok: isinstance leaves value type unknown
)
def _parse_text_block(element: object, drop_params: bool) -> _EmbeddingTextBlock:
if not isinstance(element, Mapping):
raise _invalid_input(
f"Embedding input elements must be strings, text content blocks, or file content blocks, got {type(element).__name__}"
)
block: Final = cast(Mapping[str, object], element) # cast-ok: isinstance leaves the key and value types unknown
try:
return _text_block_adapter.validate_python(_only_fields(block, _TEXT_BLOCK_FIELDS) if drop_params else block)
except ValidationError as error:
raise _invalid_input(
f"Invalid text content block in embedding input: {_validation_error_summary(error)}"
) from error
def _parse_file_block(element: object, drop_params: bool) -> _EmbeddingFileBlock:
if not isinstance(element, Mapping):
raise _invalid_input(
f"Embedding input elements must be strings or file content blocks, got {type(element).__name__}"
f"Embedding input elements must be strings, text content blocks, or file content blocks, got {type(element).__name__}"
)
block: Final = cast(Mapping[str, object], element) # cast-ok: isinstance leaves the key and value types unknown
try:
@ -268,16 +298,33 @@ def _source_mime_type(
return None
def _gcs_mime_type(gcs_url: str, mime_type_override: str | None) -> str:
"""The MIME type for a GCS file part: an explicit override, else one inferred from the URL extension.
An unresolvable extension is a client input error (400), not a connection
error (500): no request was sent when it is raised.
"""
if mime_type_override is not None:
return mime_type_override
try:
return _infer_mime_type_from_gcs_url(gcs_url)
except ValueError as error:
raise _invalid_input(str(error)) from error
def _media_part(
source: str,
mime_type_override: str | None,
resolved_files: Mapping[str, Mapping[str, str]],
) -> PartType:
if _is_data_url(source):
mime_type, base64_data = _parse_data_url(source, mime_type_override)
return PartType(inline_data=BlobType(mime_type=mime_type, data=base64_data))
try:
data_url_parts: Final = _parse_data_url(source, mime_type_override)
except ValueError as error:
raise _invalid_input(str(error)) from error
return PartType(inline_data=BlobType(mime_type=data_url_parts[0], data=data_url_parts[1]))
if _is_gcs_url(source):
gcs_mime_type: Final = mime_type_override or _infer_mime_type_from_gcs_url(source)
gcs_mime_type: Final = _gcs_mime_type(source, mime_type_override)
return PartType(file_data=FileDataType(mime_type=gcs_mime_type, file_uri=source))
if is_file_reference(source):
file_info: Final = resolved_files.get(source)
@ -299,7 +346,7 @@ def _top_level_elements(
_input_shape_adapter.validate_python(input)
except ValidationError as error:
raise _invalid_input(
f"Embedding input must be a string or a list of strings and file content blocks, got {type(input).__name__}"
f"Embedding input must be a string or a list of strings, text content blocks, and file content blocks, got {type(input).__name__}"
) from error
return [input] if isinstance(input, str) else input
@ -312,15 +359,18 @@ def _elements(input: GeminiEmbeddingInput) -> Iterator[GeminiEmbeddingElement]:
yield element
def _element_source(element: GeminiEmbeddingElement) -> str:
def _element_source(element: GeminiEmbeddingElement, drop_params: bool) -> str | None:
if isinstance(element, str):
return element
return _file_block_source(_parse_file_block(element, drop_params=True))
if _is_text_block(element):
_parse_text_block(element, drop_params)
return None
return _file_block_source(_parse_file_block(element, drop_params))
def flatten_media_sources(input: GeminiEmbeddingInput) -> tuple[str, ...]:
"""Every string element plus every file content block's source, in input order."""
return tuple(_element_source(element) for element in _elements(input))
def flatten_media_sources(input: GeminiEmbeddingInput, drop_params: bool = False) -> tuple[str, ...]:
sources: Final = (_element_source(element, drop_params) for element in _elements(input))
return tuple(source for source in sources if source is not None)
def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool:
@ -338,9 +388,9 @@ def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool:
def _is_multimodal_element(element: GeminiEmbeddingElement) -> bool:
"""Check if a single element is multimodal."""
"""Check if a single element is multimodal. Typed text blocks are text, not media."""
if not isinstance(element, str):
return True
return not _is_text_block(element)
if _is_data_url(element):
return True
if is_file_reference(element):
@ -365,6 +415,9 @@ def _build_part_for_input(
if isinstance(element, str):
return _media_part(element, None, files) if _is_multimodal_element(element) else PartType(text=element)
if _is_text_block(element):
return PartType(text=_parse_text_block(element, drop_params).text)
block: Final = _parse_file_block(element, drop_params)
part: Final = _media_part(_file_block_source(block), block.file.format, files)
if block.file.video_metadata is None:
@ -400,7 +453,7 @@ def transform_openai_input_gemini_content(
Transform OpenAI embedding input to Gemini batchEmbedContents format.
Each input element becomes a separate EmbedContentRequest, supporting
text, data URIs, file references, and GCS URLs.
text (bare strings and typed text blocks), data URIs, file references, and GCS URLs.
If an element is a list (nested input), all sub-elements are combined
into a single content with multiple parts, producing one combined
@ -421,7 +474,7 @@ def transform_openai_input_gemini_content(
for element in input_list:
if isinstance(element, list):
if not element:
raise ValueError("Nested input list must not be empty")
raise _invalid_input("Nested input list must not be empty")
parts = [
_build_part_for_input(sub, resolved_files=resolved_files, drop_params=drop_params) for sub in element
]
@ -448,7 +501,7 @@ def transform_openai_input_gemini_embed_content(
Transform OpenAI embedding input to Gemini embedContent format (multimodal).
Args:
input: GeminiEmbeddingInput with text, data URIs, or file references
input: GeminiEmbeddingInput with text (bare strings and typed text blocks), data URIs, or file references
model: Model name
optional_params: Additional parameters (taskType, outputDimensionality, etc.)
resolved_files: Dict mapping file names (files/abc) to {mime_type, uri}
@ -465,7 +518,7 @@ def transform_openai_input_gemini_embed_content(
for element in input_list:
if isinstance(element, list):
raise ValueError(
raise _invalid_input(
"Nested (combined) embeddings are not supported on the embedContent path. "
"Use the batchEmbedContents path or pass a flat list instead."
)
@ -502,6 +555,8 @@ def _is_image_element(
) -> bool:
if isinstance(element, str):
return _source_mime_type(element, None, resolved_files) in _IMAGE_MIME_TYPES
if _is_text_block(element):
return False
block: Final = _parse_file_block(element, drop_params=True)
return _source_mime_type(_file_block_source(block), block.file.format, resolved_files) in _IMAGE_MIME_TYPES
@ -514,6 +569,23 @@ def _is_image_only_input(
return bool(elements) and all(_is_image_element(element, resolved_files) for element in elements)
def _plain_text(element: GeminiEmbeddingElement) -> str | None:
if isinstance(element, str):
return None if _is_multimodal_element(element) else element
if _is_text_block(element):
return _parse_text_block(element, drop_params=True).text
return None
def _text_prompt_tokens(input: GeminiEmbeddingInput, model: str) -> int:
texts: Final = (_plain_text(element) for element in _elements(input))
text_elements: Final = [text for text in texts if text is not None]
if not text_elements:
return 0
input_text: Final = get_formatted_prompt(data={"input": text_elements}, call_type="embedding")
return token_counter(model=model, text=input_text)
def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int:
return sum(detail["tokenCount"] for detail in details if detail["modality"] == modality)
@ -521,8 +593,7 @@ def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str)
def _fallback_usage(input: GeminiEmbeddingInput, model: str) -> Usage:
if _is_multimodal_input(input):
return Usage(prompt_tokens=0, total_tokens=0)
input_text: Final = get_formatted_prompt(data={"input": input}, call_type="embedding")
prompt_tokens: Final = token_counter(model=model, text=input_text)
prompt_tokens: Final = _text_prompt_tokens(input, model)
return Usage(prompt_tokens=prompt_tokens, total_tokens=prompt_tokens)
@ -628,23 +699,7 @@ def process_response(
model_response.data = openai_embeddings
model_response.model = model
has_nested: Final = isinstance(input, list) and any(isinstance(e, list) for e in input)
if _is_multimodal_input(input) or has_nested:
input_list: Final = input if isinstance(input, list) else [input]
text_elements: Final[list[str]] = []
for e in input_list:
if isinstance(e, list):
text_elements.extend(sub for sub in e if isinstance(sub, str) and not _is_multimodal_element(sub))
elif isinstance(e, str) and not _is_multimodal_element(e):
text_elements.append(e)
if text_elements:
input_text = get_formatted_prompt(data={"input": text_elements}, call_type="embedding")
prompt_tokens = token_counter(model=model, text=input_text)
else:
prompt_tokens = 0
else:
input_text = get_formatted_prompt(data={"input": input}, call_type="embedding")
prompt_tokens = token_counter(model=model, text=input_text)
prompt_tokens: Final = _text_prompt_tokens(input, model)
model_response.usage = Usage(prompt_tokens=prompt_tokens, total_tokens=prompt_tokens)
return model_response

View file

@ -1,6 +1,6 @@
from collections.abc import Sequence
from enum import Enum
from typing import Any, Final, Literal, Protocol
from typing import Any, Final, Literal, Protocol, TypeAlias
from typing_extensions import (
ReadOnly,
@ -8,14 +8,16 @@ from typing_extensions import (
TypedDict,
)
from litellm.types.llms.openai import ChatCompletionFileObject, EmbeddingInput
from litellm.types.llms.openai import ChatCompletionFileObject, ChatCompletionTextObject, EmbeddingInput
from litellm.types.token_count import TokenCountSourceFields
# Gemini supports nested-list inputs (e.g. [["text", "image"]]) as an explicit
# opt-in for combined embeddings — a provider-specific extension of the
# opt-in for combined embeddings, and typed content blocks: a {"type": "text",
# "text": ...} block embeds its text verbatim with no media-prefix detection,
# while file blocks carry media — provider-specific extensions of the
# OpenAI-faithful EmbeddingInput shape.
GeminiEmbeddingElement = str | ChatCompletionFileObject
GeminiEmbeddingInput = (
GeminiEmbeddingElement: TypeAlias = str | ChatCompletionFileObject | ChatCompletionTextObject
GeminiEmbeddingInput: TypeAlias = (
EmbeddingInput | list[GeminiEmbeddingElement] | list[list[str]] | list[list[GeminiEmbeddingElement]]
)

View file

@ -41,7 +41,7 @@ from litellm.types.utils import (
TranscriptionResponse,
Embedding,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.llms.openai import ChatCompletionTextObject, ResponsesAPIResponse
from collections import Counter
from collections.abc import Awaitable, Callable, Mapping
from datetime import timedelta, datetime
@ -3094,6 +3094,78 @@ async def test_async_get_cache_forgets_the_worker_copy_of_a_stored_response_with
assert await handler.dual_cache.async_get_cache(key) is None
@pytest.mark.asyncio
@pytest.mark.usefixtures("copilot_response_cache")
async def test_embedding_cache_preserves_text_blocks_and_separates_them_from_media_strings() -> None:
fixed_start: Final = datetime(2026, 1, 1)
caching_handler: Final = LLMCachingHandler(original_function=aembedding, request_kwargs={}, start_time=fixed_start)
model: Final = "gemini/gemini-embedding-2"
media: Final = "data:image/png;base64,AAAA"
text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": media}
logging_obj: Final = LiteLLMLogging(
litellm_call_id="explicit-text-cache",
call_type=CallTypes.aembedding.value,
model=model,
messages=[],
function_id="explicit-text-cache",
stream=False,
start_time=fixed_start,
)
await caching_handler.async_set_cache(
result=EmbeddingResponse(model=model, data=[Embedding(embedding=[0.1], index=0, object="embedding")]),
original_function=aembedding,
kwargs={"model": model, "input": [media], "caching": True},
)
await asyncio.gather(*_PENDING_CACHE_WRITES)
partial_hit: Final = await caching_handler.async_get_cache(
model=model,
original_function=aembedding,
logging_obj=logging_obj,
start_time=fixed_start,
call_type=CallTypes.aembedding.value,
kwargs={"model": model, "input": [text_block, media], "caching": True},
)
assert partial_hit is not None
assert partial_hit.embedding_all_elements_cache_hit is False
assert partial_hit.embedding_uncached_input == [text_block]
assert partial_hit.final_embedding_cached_response is not None
assert partial_hit.final_embedding_cached_response.data[0] is None
assert partial_hit.final_embedding_cached_response.data[1].embedding == [0.1]
await caching_handler.async_set_cache(
result=EmbeddingResponse(model=model, data=[Embedding(embedding=[0.2], index=0, object="embedding")]),
original_function=aembedding,
kwargs={
"model": model,
"input": partial_hit.embedding_uncached_input,
"caching": True,
},
)
await asyncio.gather(*_PENDING_CACHE_WRITES)
full_hit: Final = await caching_handler.async_get_cache(
model=model,
original_function=aembedding,
logging_obj=logging_obj,
start_time=fixed_start,
call_type=CallTypes.aembedding.value,
kwargs={"model": model, "input": [text_block, media], "caching": True},
)
assert full_hit is not None
assert full_hit.embedding_all_elements_cache_hit is True
assert full_hit.embedding_uncached_input == []
assert full_hit.final_embedding_cached_response is not None
assert [item.embedding for item in full_hit.final_embedding_cached_response.data] == [[0.2], [0.1]]
assert [item.index for item in full_hit.final_embedding_cached_response.data] == [
0,
1,
]
@pytest.mark.asyncio
async def test_async_get_cache_partial_hit_keeps_file_block_items_uncached() -> None:
setup_cache()

View file

@ -4,16 +4,23 @@ from typing import Final
import httpx
import pytest
import respx
from google.oauth2.credentials import Credentials
import litellm
from litellm.exceptions import BadRequestError
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler import GoogleBatchEmbeddings
from litellm.types.llms.openai import ChatCompletionTextObject
from litellm.types.llms.vertex_ai import GeminiEmbeddingElement, GeminiEmbeddingInput
FILES_URI: Final = "https://generativelanguage.googleapis.com/v1beta/files/clip123"
FILE_METADATA_URL: Final = "https://generativelanguage.googleapis.com/v1beta/files/clip123"
GEMINI_BATCH_EMBEDDINGS_URL: Final = (
"https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents"
)
GEMINI_2_BATCH_EMBEDDINGS_URL: Final = (
"https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-2:batchEmbedContents"
)
def _files_api_metadata(request: httpx.Request) -> httpx.Response:
@ -67,9 +74,7 @@ async def test_gemini_embedding_sends_batch_request_and_parses_vectors(
]
}
assert [item["embedding"] for item in response.data] == [[0.1, 0.2, 0.3]]
local_token_count: Final = litellm.token_counter(
model="gemini-embedding-001", text="good morning from litellm"
)
local_token_count: Final = litellm.token_counter(model="gemini-embedding-001", text="good morning from litellm")
assert local_token_count > 0
assert (response.usage.prompt_tokens, response.usage.total_tokens) == (
local_token_count,
@ -88,3 +93,235 @@ async def test_async_resolve_file_references_fetches_the_file_metadata_for_both_
async_handler=AsyncHTTPHandler(transport=httpx.MockTransport(_files_api_metadata)),
)
assert resolved == {reference: {"mime_type": "video/mp4", "uri": FILES_URI}}
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.parametrize(
"text",
[
"data:image/png;base64,iVBORw0KGgo= is an example",
"data:application/octet-stream;base64,SGVsbG8= and surrounding prose",
"gs://my-bucket/build/output.txt",
"files/README.md is the docs entry point",
FILES_URI,
],
)
async def test_media_prefixed_text_block_embeds_verbatim_without_a_files_api_lookup(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
sync_mode: bool,
text: str,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
route: Final = respx_mock.post(GEMINI_2_BATCH_EMBEDDINGS_URL).mock(
return_value=httpx.Response(200, json={"embeddings": [{"values": [0.1, 0.2, 0.3]}]})
)
embedding_input: Final = [ChatCompletionTextObject(type="text", text=text)]
response: Final = (
litellm.embedding(model="gemini/gemini-embedding-2", input=embedding_input, api_key="gemini-test-key")
if sync_mode
else await litellm.aembedding(
model="gemini/gemini-embedding-2", input=embedding_input, api_key="gemini-test-key"
)
)
assert route.call_count == 1
assert len(respx_mock.calls) == 1
assert json.loads(route.calls[0].request.content) == {
"requests": [
{
"model": "models/gemini-embedding-2",
"content": {"parts": [{"text": text}]},
}
]
}
assert [item["embedding"] for item in response.data] == [[0.1, 0.2, 0.3]]
assert response.usage.prompt_tokens == litellm.token_counter(model="gemini-embedding-2", text=text)
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.parametrize(
"text_block",
[
{"type": "text"},
{"type": "text", "text": 123},
{"type": "text", "text": "literal text", "unexpected": True},
],
)
async def test_invalid_text_block_fails_before_resolving_other_file_references(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
sync_mode: bool,
text_block: dict[str, object],
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
embedding_input: Final = ["files/clip123", text_block]
with pytest.raises(BadRequestError, match="text") as excinfo:
(
litellm.embedding(
model="gemini/gemini-embedding-2",
input=embedding_input,
api_key="gemini-test-key",
drop_params=False,
)
if sync_mode
else await litellm.aembedding(
model="gemini/gemini-embedding-2",
input=embedding_input,
api_key="gemini-test-key",
drop_params=False,
)
)
assert excinfo.value.status_code == 400
assert len(respx_mock.calls) == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_mode", [True, False])
async def test_drop_params_ignores_extra_text_fields_and_preserves_other_file_references(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
sync_mode: bool,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
metadata_route: Final = respx_mock.get(FILE_METADATA_URL).mock(
return_value=httpx.Response(200, json={"mimeType": "video/mp4", "uri": FILES_URI})
)
route: Final = respx_mock.post(GEMINI_2_BATCH_EMBEDDINGS_URL).mock(
return_value=httpx.Response(200, json={"embeddings": [{"values": [0.1]}, {"values": [0.2]}]})
)
embedding_input: Final = [
{"type": "text", "text": "files/README.md", "unexpected": True},
"files/clip123",
]
response: Final = (
litellm.embedding(
model="gemini/gemini-embedding-2", input=embedding_input, api_key="gemini-test-key", drop_params=True
)
if sync_mode
else await litellm.aembedding(
model="gemini/gemini-embedding-2", input=embedding_input, api_key="gemini-test-key", drop_params=True
)
)
assert metadata_route.call_count == 1
assert route.call_count == 1
assert len(respx_mock.calls) == 2
assert json.loads(route.calls[0].request.content) == {
"requests": [
{"model": "models/gemini-embedding-2", "content": {"parts": [{"text": "files/README.md"}]}},
{
"model": "models/gemini-embedding-2",
"content": {"parts": [{"file_data": {"mime_type": "video/mp4", "file_uri": FILES_URI}}]},
},
]
}
assert [item["embedding"] for item in response.data] == [[0.1], [0.2]]
@pytest.mark.asyncio
@pytest.mark.parametrize("combined", [False, True])
async def test_text_and_legacy_media_preserve_embedding_groups_and_order(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
combined: bool,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
media: Final = "data:image/png;base64,iVBORw0KGgo="
elements: Final[list[GeminiEmbeddingElement]] = [ChatCompletionTextObject(type="text", text=media), media]
embedding_input: Final[GeminiEmbeddingInput] = [elements] if combined else elements
vectors: Final = [[0.1]] if combined else [[0.1], [0.2]]
route: Final = respx_mock.post(GEMINI_2_BATCH_EMBEDDINGS_URL).mock(
return_value=httpx.Response(200, json={"embeddings": [{"values": vector} for vector in vectors]})
)
response: Final = await litellm.aembedding(
model="gemini/gemini-embedding-2", input=embedding_input, api_key="gemini-test-key"
)
parts: Final = [{"text": media}, {"inline_data": {"mime_type": "image/png", "data": "iVBORw0KGgo="}}]
groups: Final = [parts] if combined else [[part] for part in parts]
assert json.loads(route.calls[0].request.content) == {
"requests": [{"model": "models/gemini-embedding-2", "content": {"parts": group}} for group in groups]
}
assert [item["embedding"] for item in response.data] == vectors
assert [item["index"] for item in response.data] == list(range(len(vectors)))
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_mode", [True, False])
async def test_vertex_embedding_sends_media_prefixed_text_as_literal_parts(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
sync_mode: bool,
) -> None:
from litellm.main import google_batch_embeddings
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
credentials_key: Final = "test-vertex-embedding-credentials"
monkeypatch.setitem(
google_batch_embeddings._credentials_project_mapping,
(credentials_key, "test-project"),
(Credentials(token="test-vertex-token"), "test-project"),
)
url: Final = (
"https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/"
"locations/us-central1/publishers/google/models/gemini-embedding-2:embedContent"
)
route: Final = respx_mock.post(url).mock(
return_value=httpx.Response(200, json={"embedding": {"values": [0.1, 0.2]}})
)
text: Final = "gs://my-bucket/build/output.txt"
embedding_input: Final = [ChatCompletionTextObject(type="text", text=text)]
response: Final = (
litellm.embedding(
model="vertex_ai/gemini-embedding-2",
input=embedding_input,
vertex_project="test-project",
vertex_location="us-central1",
vertex_credentials=credentials_key,
)
if sync_mode
else await litellm.aembedding(
model="vertex_ai/gemini-embedding-2",
input=embedding_input,
vertex_project="test-project",
vertex_location="us-central1",
vertex_credentials=credentials_key,
)
)
assert route.call_count == 1
assert len(respx_mock.calls) == 1
assert json.loads(route.calls[0].request.content) == {"content": {"parts": [{"text": text}]}}
assert [item["embedding"] for item in response.data] == [[0.1, 0.2]]
assert response.usage.prompt_tokens == litellm.token_counter(model="gemini-embedding-2", text=text)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"bad_input",
[
["data:application/octet-stream;base64,SGVsbG8=", "plain text chunk"],
["gs://my-bucket/build/output.txt", "plain text chunk"],
],
)
async def test_media_prefix_validation_fails_as_bad_request_without_sending_a_request(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
bad_input: list[str],
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with pytest.raises(BadRequestError) as excinfo:
await litellm.aembedding(model="gemini/gemini-embedding-2", input=bad_input, api_key="gemini-test-key")
assert excinfo.value.status_code == 400
assert len(respx_mock.calls) == 0

View file

@ -21,8 +21,10 @@ from litellm.exceptions import BadRequestError
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import (
_build_part_for_input,
file_reference_name,
_is_multimodal_input,
_source_mime_type,
file_reference_name,
flatten_media_sources,
process_embed_content_response,
process_response,
transform_openai_input_gemini_content,
@ -305,7 +307,8 @@ class TestProcessResponse:
assert result.usage.prompt_tokens > 0
def test_nested_empty_list_raises(self):
with pytest.raises(ValueError, match="must not be empty"):
# local input validation: a 400 BadRequestError, not a ValueError that surfaces as a 500
with pytest.raises(BadRequestError, match="must not be empty"):
transform_openai_input_gemini_content(
input=[[]],
model="gemini-embedding-2-preview",
@ -313,7 +316,9 @@ class TestProcessResponse:
)
def test_nested_non_string_element_raises(self):
with pytest.raises(BadRequestError, match="must be strings or file content blocks, got list"):
with pytest.raises(
BadRequestError, match="must be strings, text content blocks, or file content blocks, got list"
):
transform_openai_input_gemini_content(
input=[[["doubly", "nested"]]],
model="gemini-embedding-2-preview",
@ -519,7 +524,8 @@ class TestFileContentBlocks:
assert parts[1] == {"text": "a solid color clip"}
def test_embed_content_path_still_rejects_nested_lists(self) -> None:
with pytest.raises(ValueError, match="Nested"):
# local input validation: a 400 BadRequestError, not a ValueError that surfaces as a 500
with pytest.raises(BadRequestError, match="Nested"):
transform_openai_input_gemini_embed_content(
input=[[_file_block(file_data=VIDEO_DATA_URI), "a solid color clip"]],
model=self.MODEL,
@ -548,9 +554,7 @@ class TestFileContentBlocks:
({"type": "image_url", "image_url": {"url": IMAGE_DATA_URI}}, "Input should be 'file'"),
],
)
def test_malformed_block_answers_400_naming_the_field(
self, block: dict[str, object], named_in_error: str
) -> None:
def test_malformed_block_answers_400_naming_the_field(self, block: dict[str, object], named_in_error: str) -> None:
with pytest.raises(BadRequestError, match=named_in_error):
_build_part_for_input(block)
@ -652,3 +656,178 @@ def test_gemini_embedding(respx_mock: respx.MockRouter) -> None:
expected_prompt_tokens: Final = litellm.token_counter(model="gemini/gemini-embedding-001", text="Hello, world!")
assert response.usage.prompt_tokens == expected_prompt_tokens
assert response.usage.total_tokens == expected_prompt_tokens
TEXT_BLOCK_DATA_PREFIX: Final = {
"type": "text",
"text": "data:application/octet-stream;base64,SGVsbG8= and surrounding prose",
}
TEXT_BLOCK_GCS: Final = {"type": "text", "text": "gs://my-bucket/build/output.txt"}
TEXT_BLOCK_FILES: Final = {"type": "text", "text": "files/README.md is the docs entry point"}
def _text_block(**overrides: object) -> dict[str, object]:
block: Final = {"type": "text", "text": "some plain text"}
block.update(overrides)
return block
class TestTypedTextBlocks:
"""A {"type": "text"} block is embedded verbatim: no media prefix detection runs on it.
Regression for #46719: text chunks that happen to start with data:, gs:// or
files/ were read as media, with no way for the caller to say they are text.
"""
def test_data_url_prefixed_text_is_embedded_as_text(self) -> None:
part: Final = _build_part_for_input(TEXT_BLOCK_DATA_PREFIX)
assert part == {"text": TEXT_BLOCK_DATA_PREFIX["text"]}
def test_gcs_prefixed_text_is_embedded_as_text(self) -> None:
part: Final = _build_part_for_input(TEXT_BLOCK_GCS)
assert part == {"text": TEXT_BLOCK_GCS["text"]}
def test_files_prefixed_text_is_embedded_as_text(self) -> None:
part: Final = _build_part_for_input(TEXT_BLOCK_FILES)
assert part == {"text": TEXT_BLOCK_FILES["text"]}
def test_text_block_keeps_detection_for_other_elements(self) -> None:
parts: Final = [_build_part_for_input(element) for element in [TEXT_BLOCK_DATA_PREFIX, IMAGE_DATA_URI]]
assert parts == [
{"text": TEXT_BLOCK_DATA_PREFIX["text"]},
{"inline_data": {"mime_type": "image/png", "data": IMAGE_DATA_URI.split(",", 1)[1]}},
]
def test_text_block_is_not_multimodal(self) -> None:
assert _is_multimodal_input([_text_block(text="data:image/png;base64,iVBORw0KGgo=")]) is False
def test_mixed_input_with_text_block_is_multimodal(self) -> None:
assert _is_multimodal_input([_text_block(), IMAGE_DATA_URI]) is True
def test_empty_text_block_embeds_like_an_empty_string(self) -> None:
assert _build_part_for_input(_text_block(text="")) == {"text": ""}
def test_text_block_with_extra_keys_is_rejected(self) -> None:
with pytest.raises(BadRequestError):
_build_part_for_input(_text_block(cache_control={"type": "ephemeral"}))
def test_text_block_with_extra_keys_is_dropped_under_drop_params(self) -> None:
part: Final = _build_part_for_input(_text_block(cache_control={"type": "ephemeral"}), drop_params=True)
assert part == {"text": "some plain text"}
def test_text_block_missing_text_is_rejected(self) -> None:
with pytest.raises(BadRequestError):
_build_part_for_input({"type": "text"})
def test_text_block_non_string_text_is_rejected(self) -> None:
with pytest.raises(BadRequestError):
_build_part_for_input(_text_block(text=123))
def test_nested_group_with_text_block_builds_one_combined_request(self) -> None:
body: Final = transform_openai_input_gemini_content(
input=[[TEXT_BLOCK_GCS, "plain chunk"]],
model="gemini-embedding-001",
optional_params={},
)
assert body["requests"] == [
{
"model": "models/gemini-embedding-001",
"content": {"parts": [{"text": TEXT_BLOCK_GCS["text"]}, {"text": "plain chunk"}]},
}
]
class TestTextBlocksAreNeverFileReferences:
def test_flatten_media_sources_skips_text_blocks(self) -> None:
sources: Final = flatten_media_sources([TEXT_BLOCK_FILES, "a red bus"])
assert sources == ("a red bus",)
class TestLocalValidationIsClientError:
"""Local input validation failures are BadRequestError (400), not ValueError-turned-500.
No request was sent when they raise, so surfacing them as APIConnectionError
makes retry logic retry a permanent input error until its time limit (#46719).
"""
def test_unsupported_data_url_mime_is_a_client_error(self) -> None:
with pytest.raises(BadRequestError) as excinfo:
transform_openai_input_gemini_content(
input=["data:application/octet-stream;base64,SGVsbG8="],
model="gemini-embedding-001",
optional_params={},
)
assert "Unsupported MIME type for embedding" in str(excinfo.value)
def test_gcs_unresolvable_extension_is_a_client_error(self) -> None:
with pytest.raises(BadRequestError) as excinfo:
transform_openai_input_gemini_content(
input=["gs://my-bucket/build/output.txt"],
model="gemini-embedding-001",
optional_params={},
)
assert "Unable to infer MIME type from GCS URL" in str(excinfo.value)
def test_gcs_mime_type_override_skips_inference(self) -> None:
part: Final = _build_part_for_input(_file_block(file_id="gs://my-bucket/build/output.txt", format="text/plain"))
assert part == {"file_data": {"mime_type": "text/plain", "file_uri": "gs://my-bucket/build/output.txt"}}
def test_source_mime_type_soft_probe_still_returns_none(self) -> None:
assert _source_mime_type("data:application/octet-stream;base64,SGVsbG8=", None, {}) is None
assert _source_mime_type("gs://my-bucket/build/output.txt", None, {}) is None
def test_empty_nested_group_is_a_client_error(self) -> None:
with pytest.raises(BadRequestError):
transform_openai_input_gemini_content(
input=[["a red bus"], []],
model="gemini-embedding-001",
optional_params={},
)
def test_embed_content_nested_input_is_a_client_error(self) -> None:
with pytest.raises(BadRequestError):
transform_openai_input_gemini_embed_content(
input=[["a red bus", IMAGE_DATA_URI]],
model="gemini-embedding-001",
optional_params={},
)
class TestTextBlockTokenAccounting:
"""A text block must count its text exactly like the same text as a bare string."""
def _predictions(self) -> VertexAIBatchEmbeddingsResponseObject:
return VertexAIBatchEmbeddingsResponseObject(embeddings=[{"values": [0.1, 0.2]}, {"values": [0.3, 0.4]}])
def test_text_block_counts_like_the_same_bare_string(self) -> None:
chunk: Final = "the quick brown fox starts a chunk"
via_block: Final = process_response(
input=[_text_block(text=chunk), IMAGE_DATA_URI],
model_response=EmbeddingResponse(),
model="gemini-embedding-001",
_predictions=self._predictions(),
)
via_string: Final = process_response(
input=[chunk, IMAGE_DATA_URI],
model_response=EmbeddingResponse(),
model="gemini-embedding-001",
_predictions=self._predictions(),
)
assert via_block.usage.prompt_tokens == via_string.usage.prompt_tokens
assert via_block.usage.prompt_tokens > 0
def test_embed_content_fallback_counts_text_block_like_a_bare_string(self) -> None:
text: Final = "hello world this is plain text"
via_block: Final = process_embed_content_response(
input=[_text_block(text=text)],
model_response=EmbeddingResponse(),
model="gemini-embedding-001",
response_json={"embedding": {"values": [0.1, 0.2]}},
)
via_string: Final = process_embed_content_response(
input=[text],
model_response=EmbeddingResponse(),
model="gemini-embedding-001",
response_json={"embedding": {"values": [0.1, 0.2]}},
)
assert via_block.usage.prompt_tokens == via_string.usage.prompt_tokens > 0