diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 7089ff78835..a8174705e9b 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -39,7 +39,7 @@ from litellm.litellm_core_utils.logging_utils import ( from litellm.types.caching import CACHED_STREAM_EVENTS_KEY, EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding from litellm.types.integrations.custom_logger import converted_stream_requested from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.llms.openai import ChatCompletionFileObject, ResponsesAPIResponse +from litellm.types.llms.openai import ChatCompletionFileObject, ChatCompletionTextObject, ResponsesAPIResponse from litellm.types.rerank import RerankResponse from litellm.types.utils import ( CachingDetails, @@ -71,7 +71,7 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -EmbeddingCacheInputElement = str | list[int] | ChatCompletionFileObject +EmbeddingCacheInputElement = str | list[int] | ChatCompletionFileObject | ChatCompletionTextObject class CachingHandlerResponse(LiteLLMBaseModel): diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py index f15c6393cb2..a52af174632 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py @@ -45,9 +45,10 @@ class GoogleBatchEmbeddings(VertexLLM): @staticmethod def _flatten_and_detect_file_refs( input: GeminiEmbeddingInput, + drop_params: bool = False, ) -> tuple[list[str], bool]: """Flatten nested input lists and file content blocks into their sources and detect file references.""" - flat_elements: Final = list(flatten_media_sources(input)) + flat_elements: Final = list(flatten_media_sources(input, drop_params=drop_params)) has_file_refs: Final = any(is_file_reference(e) for e in flat_elements) return flat_elements, has_file_refs @@ -218,7 +219,7 @@ class GoogleBatchEmbeddings(VertexLLM): ### TRANSFORMATION (sync path) ### request_data: VertexAIBatchEmbeddingsRequestBody | dict[str, object] - flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input) + flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input, drop_params=drop_params) if use_embed_content: resolved_files = {} if api_key: @@ -324,7 +325,7 @@ class GoogleBatchEmbeddings(VertexLLM): async_handler = client ### TRANSFORMATION (async path) ### - flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input) + flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input, drop_params=drop_params) if use_embed_content: resolved_files = {} if api_key: diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 96a785c06c9..5ccfcd56af1 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -166,11 +166,20 @@ class _EmbeddingFileBlock(LiteLLMBaseModel): file: _EmbeddingFile +class _EmbeddingTextBlock(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + type: Literal["text"] + text: str + + _file_block_adapter: Final = TypeAdapter(_EmbeddingFileBlock) +_text_block_adapter: Final = TypeAdapter(_EmbeddingTextBlock) _video_metadata_adapter: Final = TypeAdapter(VideoMetadataType) _input_shape_adapter: Final[TypeAdapter[str | list[object]]] = TypeAdapter(str | list[object]) _mapping_adapter: Final = TypeAdapter(dict[str, object]) _BLOCK_FIELDS: Final = frozenset(_EmbeddingFileBlock.model_fields) +_TEXT_BLOCK_FIELDS: Final = frozenset(_EmbeddingTextBlock.model_fields) _FILE_FIELDS: Final = frozenset(_EmbeddingFile.model_fields) _VIDEO_METADATA_FIELDS: Final = frozenset(_EmbeddingVideoMetadata.model_fields) _FILE_SOURCE_FORMS: Final = "a data: URI, a gs:// URL, a files/ reference, or a Gemini Files API URI" @@ -211,10 +220,31 @@ def _dropping_unsupported_keys(block: Mapping[str, object]) -> dict[str, object] } +def _is_text_block(element: object) -> bool: + return ( + isinstance(element, Mapping) + and cast(Mapping[str, object], element).get("type") == "text" # cast-ok: isinstance leaves value type unknown + ) + + +def _parse_text_block(element: object, drop_params: bool) -> _EmbeddingTextBlock: + if not isinstance(element, Mapping): + raise _invalid_input( + f"Embedding input elements must be strings, text content blocks, or file content blocks, got {type(element).__name__}" + ) + block: Final = cast(Mapping[str, object], element) # cast-ok: isinstance leaves the key and value types unknown + try: + return _text_block_adapter.validate_python(_only_fields(block, _TEXT_BLOCK_FIELDS) if drop_params else block) + except ValidationError as error: + raise _invalid_input( + f"Invalid text content block in embedding input: {_validation_error_summary(error)}" + ) from error + + def _parse_file_block(element: object, drop_params: bool) -> _EmbeddingFileBlock: if not isinstance(element, Mapping): raise _invalid_input( - f"Embedding input elements must be strings or file content blocks, got {type(element).__name__}" + f"Embedding input elements must be strings, text content blocks, or file content blocks, got {type(element).__name__}" ) block: Final = cast(Mapping[str, object], element) # cast-ok: isinstance leaves the key and value types unknown try: @@ -268,16 +298,33 @@ def _source_mime_type( return None +def _gcs_mime_type(gcs_url: str, mime_type_override: str | None) -> str: + """The MIME type for a GCS file part: an explicit override, else one inferred from the URL extension. + + An unresolvable extension is a client input error (400), not a connection + error (500): no request was sent when it is raised. + """ + if mime_type_override is not None: + return mime_type_override + try: + return _infer_mime_type_from_gcs_url(gcs_url) + except ValueError as error: + raise _invalid_input(str(error)) from error + + def _media_part( source: str, mime_type_override: str | None, resolved_files: Mapping[str, Mapping[str, str]], ) -> PartType: if _is_data_url(source): - mime_type, base64_data = _parse_data_url(source, mime_type_override) - return PartType(inline_data=BlobType(mime_type=mime_type, data=base64_data)) + try: + data_url_parts: Final = _parse_data_url(source, mime_type_override) + except ValueError as error: + raise _invalid_input(str(error)) from error + return PartType(inline_data=BlobType(mime_type=data_url_parts[0], data=data_url_parts[1])) if _is_gcs_url(source): - gcs_mime_type: Final = mime_type_override or _infer_mime_type_from_gcs_url(source) + gcs_mime_type: Final = _gcs_mime_type(source, mime_type_override) return PartType(file_data=FileDataType(mime_type=gcs_mime_type, file_uri=source)) if is_file_reference(source): file_info: Final = resolved_files.get(source) @@ -299,7 +346,7 @@ def _top_level_elements( _input_shape_adapter.validate_python(input) except ValidationError as error: raise _invalid_input( - f"Embedding input must be a string or a list of strings and file content blocks, got {type(input).__name__}" + f"Embedding input must be a string or a list of strings, text content blocks, and file content blocks, got {type(input).__name__}" ) from error return [input] if isinstance(input, str) else input @@ -312,15 +359,18 @@ def _elements(input: GeminiEmbeddingInput) -> Iterator[GeminiEmbeddingElement]: yield element -def _element_source(element: GeminiEmbeddingElement) -> str: +def _element_source(element: GeminiEmbeddingElement, drop_params: bool) -> str | None: if isinstance(element, str): return element - return _file_block_source(_parse_file_block(element, drop_params=True)) + if _is_text_block(element): + _parse_text_block(element, drop_params) + return None + return _file_block_source(_parse_file_block(element, drop_params)) -def flatten_media_sources(input: GeminiEmbeddingInput) -> tuple[str, ...]: - """Every string element plus every file content block's source, in input order.""" - return tuple(_element_source(element) for element in _elements(input)) +def flatten_media_sources(input: GeminiEmbeddingInput, drop_params: bool = False) -> tuple[str, ...]: + sources: Final = (_element_source(element, drop_params) for element in _elements(input)) + return tuple(source for source in sources if source is not None) def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool: @@ -338,9 +388,9 @@ def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool: def _is_multimodal_element(element: GeminiEmbeddingElement) -> bool: - """Check if a single element is multimodal.""" + """Check if a single element is multimodal. Typed text blocks are text, not media.""" if not isinstance(element, str): - return True + return not _is_text_block(element) if _is_data_url(element): return True if is_file_reference(element): @@ -365,6 +415,9 @@ def _build_part_for_input( if isinstance(element, str): return _media_part(element, None, files) if _is_multimodal_element(element) else PartType(text=element) + if _is_text_block(element): + return PartType(text=_parse_text_block(element, drop_params).text) + block: Final = _parse_file_block(element, drop_params) part: Final = _media_part(_file_block_source(block), block.file.format, files) if block.file.video_metadata is None: @@ -400,7 +453,7 @@ def transform_openai_input_gemini_content( Transform OpenAI embedding input to Gemini batchEmbedContents format. Each input element becomes a separate EmbedContentRequest, supporting - text, data URIs, file references, and GCS URLs. + text (bare strings and typed text blocks), data URIs, file references, and GCS URLs. If an element is a list (nested input), all sub-elements are combined into a single content with multiple parts, producing one combined @@ -421,7 +474,7 @@ def transform_openai_input_gemini_content( for element in input_list: if isinstance(element, list): if not element: - raise ValueError("Nested input list must not be empty") + raise _invalid_input("Nested input list must not be empty") parts = [ _build_part_for_input(sub, resolved_files=resolved_files, drop_params=drop_params) for sub in element ] @@ -448,7 +501,7 @@ def transform_openai_input_gemini_embed_content( Transform OpenAI embedding input to Gemini embedContent format (multimodal). Args: - input: GeminiEmbeddingInput with text, data URIs, or file references + input: GeminiEmbeddingInput with text (bare strings and typed text blocks), data URIs, or file references model: Model name optional_params: Additional parameters (taskType, outputDimensionality, etc.) resolved_files: Dict mapping file names (files/abc) to {mime_type, uri} @@ -465,7 +518,7 @@ def transform_openai_input_gemini_embed_content( for element in input_list: if isinstance(element, list): - raise ValueError( + raise _invalid_input( "Nested (combined) embeddings are not supported on the embedContent path. " "Use the batchEmbedContents path or pass a flat list instead." ) @@ -502,6 +555,8 @@ def _is_image_element( ) -> bool: if isinstance(element, str): return _source_mime_type(element, None, resolved_files) in _IMAGE_MIME_TYPES + if _is_text_block(element): + return False block: Final = _parse_file_block(element, drop_params=True) return _source_mime_type(_file_block_source(block), block.file.format, resolved_files) in _IMAGE_MIME_TYPES @@ -514,6 +569,23 @@ def _is_image_only_input( return bool(elements) and all(_is_image_element(element, resolved_files) for element in elements) +def _plain_text(element: GeminiEmbeddingElement) -> str | None: + if isinstance(element, str): + return None if _is_multimodal_element(element) else element + if _is_text_block(element): + return _parse_text_block(element, drop_params=True).text + return None + + +def _text_prompt_tokens(input: GeminiEmbeddingInput, model: str) -> int: + texts: Final = (_plain_text(element) for element in _elements(input)) + text_elements: Final = [text for text in texts if text is not None] + if not text_elements: + return 0 + input_text: Final = get_formatted_prompt(data={"input": text_elements}, call_type="embedding") + return token_counter(model=model, text=input_text) + + def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int: return sum(detail["tokenCount"] for detail in details if detail["modality"] == modality) @@ -521,8 +593,7 @@ def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) def _fallback_usage(input: GeminiEmbeddingInput, model: str) -> Usage: if _is_multimodal_input(input): return Usage(prompt_tokens=0, total_tokens=0) - input_text: Final = get_formatted_prompt(data={"input": input}, call_type="embedding") - prompt_tokens: Final = token_counter(model=model, text=input_text) + prompt_tokens: Final = _text_prompt_tokens(input, model) return Usage(prompt_tokens=prompt_tokens, total_tokens=prompt_tokens) @@ -628,23 +699,7 @@ def process_response( model_response.data = openai_embeddings model_response.model = model - has_nested: Final = isinstance(input, list) and any(isinstance(e, list) for e in input) - if _is_multimodal_input(input) or has_nested: - input_list: Final = input if isinstance(input, list) else [input] - text_elements: Final[list[str]] = [] - for e in input_list: - if isinstance(e, list): - text_elements.extend(sub for sub in e if isinstance(sub, str) and not _is_multimodal_element(sub)) - elif isinstance(e, str) and not _is_multimodal_element(e): - text_elements.append(e) - if text_elements: - input_text = get_formatted_prompt(data={"input": text_elements}, call_type="embedding") - prompt_tokens = token_counter(model=model, text=input_text) - else: - prompt_tokens = 0 - else: - input_text = get_formatted_prompt(data={"input": input}, call_type="embedding") - prompt_tokens = token_counter(model=model, text=input_text) + prompt_tokens: Final = _text_prompt_tokens(input, model) model_response.usage = Usage(prompt_tokens=prompt_tokens, total_tokens=prompt_tokens) return model_response diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index f1176f777c7..eae76faf787 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -1,6 +1,6 @@ from collections.abc import Sequence from enum import Enum -from typing import Any, Final, Literal, Protocol +from typing import Any, Final, Literal, Protocol, TypeAlias from typing_extensions import ( ReadOnly, @@ -8,14 +8,16 @@ from typing_extensions import ( TypedDict, ) -from litellm.types.llms.openai import ChatCompletionFileObject, EmbeddingInput +from litellm.types.llms.openai import ChatCompletionFileObject, ChatCompletionTextObject, EmbeddingInput from litellm.types.token_count import TokenCountSourceFields # Gemini supports nested-list inputs (e.g. [["text", "image"]]) as an explicit -# opt-in for combined embeddings — a provider-specific extension of the +# opt-in for combined embeddings, and typed content blocks: a {"type": "text", +# "text": ...} block embeds its text verbatim with no media-prefix detection, +# while file blocks carry media — provider-specific extensions of the # OpenAI-faithful EmbeddingInput shape. -GeminiEmbeddingElement = str | ChatCompletionFileObject -GeminiEmbeddingInput = ( +GeminiEmbeddingElement: TypeAlias = str | ChatCompletionFileObject | ChatCompletionTextObject +GeminiEmbeddingInput: TypeAlias = ( EmbeddingInput | list[GeminiEmbeddingElement] | list[list[str]] | list[list[GeminiEmbeddingElement]] ) diff --git a/tests/unit/caching/test_caching_handler.py b/tests/unit/caching/test_caching_handler.py index 51754c80dd6..33439206dcf 100644 --- a/tests/unit/caching/test_caching_handler.py +++ b/tests/unit/caching/test_caching_handler.py @@ -41,7 +41,7 @@ from litellm.types.utils import ( TranscriptionResponse, Embedding, ) -from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.llms.openai import ChatCompletionTextObject, ResponsesAPIResponse from collections import Counter from collections.abc import Awaitable, Callable, Mapping from datetime import timedelta, datetime @@ -3094,6 +3094,78 @@ async def test_async_get_cache_forgets_the_worker_copy_of_a_stored_response_with assert await handler.dual_cache.async_get_cache(key) is None +@pytest.mark.asyncio +@pytest.mark.usefixtures("copilot_response_cache") +async def test_embedding_cache_preserves_text_blocks_and_separates_them_from_media_strings() -> None: + fixed_start: Final = datetime(2026, 1, 1) + caching_handler: Final = LLMCachingHandler(original_function=aembedding, request_kwargs={}, start_time=fixed_start) + model: Final = "gemini/gemini-embedding-2" + media: Final = "data:image/png;base64,AAAA" + text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": media} + logging_obj: Final = LiteLLMLogging( + litellm_call_id="explicit-text-cache", + call_type=CallTypes.aembedding.value, + model=model, + messages=[], + function_id="explicit-text-cache", + stream=False, + start_time=fixed_start, + ) + await caching_handler.async_set_cache( + result=EmbeddingResponse(model=model, data=[Embedding(embedding=[0.1], index=0, object="embedding")]), + original_function=aembedding, + kwargs={"model": model, "input": [media], "caching": True}, + ) + await asyncio.gather(*_PENDING_CACHE_WRITES) + + partial_hit: Final = await caching_handler.async_get_cache( + model=model, + original_function=aembedding, + logging_obj=logging_obj, + start_time=fixed_start, + call_type=CallTypes.aembedding.value, + kwargs={"model": model, "input": [text_block, media], "caching": True}, + ) + + assert partial_hit is not None + assert partial_hit.embedding_all_elements_cache_hit is False + assert partial_hit.embedding_uncached_input == [text_block] + assert partial_hit.final_embedding_cached_response is not None + assert partial_hit.final_embedding_cached_response.data[0] is None + assert partial_hit.final_embedding_cached_response.data[1].embedding == [0.1] + + await caching_handler.async_set_cache( + result=EmbeddingResponse(model=model, data=[Embedding(embedding=[0.2], index=0, object="embedding")]), + original_function=aembedding, + kwargs={ + "model": model, + "input": partial_hit.embedding_uncached_input, + "caching": True, + }, + ) + await asyncio.gather(*_PENDING_CACHE_WRITES) + + full_hit: Final = await caching_handler.async_get_cache( + model=model, + original_function=aembedding, + logging_obj=logging_obj, + start_time=fixed_start, + call_type=CallTypes.aembedding.value, + kwargs={"model": model, "input": [text_block, media], "caching": True}, + ) + + assert full_hit is not None + assert full_hit.embedding_all_elements_cache_hit is True + assert full_hit.embedding_uncached_input == [] + assert full_hit.final_embedding_cached_response is not None + assert [item.embedding for item in full_hit.final_embedding_cached_response.data] == [[0.2], [0.1]] + assert [item.index for item in full_hit.final_embedding_cached_response.data] == [ + 0, + 1, + ] + + + @pytest.mark.asyncio async def test_async_get_cache_partial_hit_keeps_file_block_items_uncached() -> None: setup_cache() diff --git a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py index 231bc25a7f0..6693abb4e28 100644 --- a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py +++ b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py @@ -4,16 +4,23 @@ from typing import Final import httpx import pytest import respx +from google.oauth2.credentials import Credentials import litellm +from litellm.exceptions import BadRequestError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler import GoogleBatchEmbeddings +from litellm.types.llms.openai import ChatCompletionTextObject +from litellm.types.llms.vertex_ai import GeminiEmbeddingElement, GeminiEmbeddingInput FILES_URI: Final = "https://generativelanguage.googleapis.com/v1beta/files/clip123" FILE_METADATA_URL: Final = "https://generativelanguage.googleapis.com/v1beta/files/clip123" GEMINI_BATCH_EMBEDDINGS_URL: Final = ( "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:batchEmbedContents" ) +GEMINI_2_BATCH_EMBEDDINGS_URL: Final = ( + "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-2:batchEmbedContents" +) def _files_api_metadata(request: httpx.Request) -> httpx.Response: @@ -67,9 +74,7 @@ async def test_gemini_embedding_sends_batch_request_and_parses_vectors( ] } assert [item["embedding"] for item in response.data] == [[0.1, 0.2, 0.3]] - local_token_count: Final = litellm.token_counter( - model="gemini-embedding-001", text="good morning from litellm" - ) + local_token_count: Final = litellm.token_counter(model="gemini-embedding-001", text="good morning from litellm") assert local_token_count > 0 assert (response.usage.prompt_tokens, response.usage.total_tokens) == ( local_token_count, @@ -88,3 +93,235 @@ async def test_async_resolve_file_references_fetches_the_file_metadata_for_both_ async_handler=AsyncHTTPHandler(transport=httpx.MockTransport(_files_api_metadata)), ) assert resolved == {reference: {"mime_type": "video/mp4", "uri": FILES_URI}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize( + "text", + [ + "data:image/png;base64,iVBORw0KGgo= is an example", + "data:application/octet-stream;base64,SGVsbG8= and surrounding prose", + "gs://my-bucket/build/output.txt", + "files/README.md is the docs entry point", + FILES_URI, + ], +) +async def test_media_prefixed_text_block_embeds_verbatim_without_a_files_api_lookup( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + text: str, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post(GEMINI_2_BATCH_EMBEDDINGS_URL).mock( + return_value=httpx.Response(200, json={"embeddings": [{"values": [0.1, 0.2, 0.3]}]}) + ) + + embedding_input: Final = [ChatCompletionTextObject(type="text", text=text)] + response: Final = ( + litellm.embedding(model="gemini/gemini-embedding-2", input=embedding_input, api_key="gemini-test-key") + if sync_mode + else await litellm.aembedding( + model="gemini/gemini-embedding-2", input=embedding_input, api_key="gemini-test-key" + ) + ) + + assert route.call_count == 1 + assert len(respx_mock.calls) == 1 + assert json.loads(route.calls[0].request.content) == { + "requests": [ + { + "model": "models/gemini-embedding-2", + "content": {"parts": [{"text": text}]}, + } + ] + } + assert [item["embedding"] for item in response.data] == [[0.1, 0.2, 0.3]] + assert response.usage.prompt_tokens == litellm.token_counter(model="gemini-embedding-2", text=text) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize( + "text_block", + [ + {"type": "text"}, + {"type": "text", "text": 123}, + {"type": "text", "text": "literal text", "unexpected": True}, + ], +) +async def test_invalid_text_block_fails_before_resolving_other_file_references( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + text_block: dict[str, object], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + embedding_input: Final = ["files/clip123", text_block] + + with pytest.raises(BadRequestError, match="text") as excinfo: + ( + litellm.embedding( + model="gemini/gemini-embedding-2", + input=embedding_input, + api_key="gemini-test-key", + drop_params=False, + ) + if sync_mode + else await litellm.aembedding( + model="gemini/gemini-embedding-2", + input=embedding_input, + api_key="gemini-test-key", + drop_params=False, + ) + ) + + assert excinfo.value.status_code == 400 + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_drop_params_ignores_extra_text_fields_and_preserves_other_file_references( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + metadata_route: Final = respx_mock.get(FILE_METADATA_URL).mock( + return_value=httpx.Response(200, json={"mimeType": "video/mp4", "uri": FILES_URI}) + ) + route: Final = respx_mock.post(GEMINI_2_BATCH_EMBEDDINGS_URL).mock( + return_value=httpx.Response(200, json={"embeddings": [{"values": [0.1]}, {"values": [0.2]}]}) + ) + embedding_input: Final = [ + {"type": "text", "text": "files/README.md", "unexpected": True}, + "files/clip123", + ] + + response: Final = ( + litellm.embedding( + model="gemini/gemini-embedding-2", input=embedding_input, api_key="gemini-test-key", drop_params=True + ) + if sync_mode + else await litellm.aembedding( + model="gemini/gemini-embedding-2", input=embedding_input, api_key="gemini-test-key", drop_params=True + ) + ) + + assert metadata_route.call_count == 1 + assert route.call_count == 1 + assert len(respx_mock.calls) == 2 + assert json.loads(route.calls[0].request.content) == { + "requests": [ + {"model": "models/gemini-embedding-2", "content": {"parts": [{"text": "files/README.md"}]}}, + { + "model": "models/gemini-embedding-2", + "content": {"parts": [{"file_data": {"mime_type": "video/mp4", "file_uri": FILES_URI}}]}, + }, + ] + } + assert [item["embedding"] for item in response.data] == [[0.1], [0.2]] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("combined", [False, True]) +async def test_text_and_legacy_media_preserve_embedding_groups_and_order( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + combined: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + media: Final = "data:image/png;base64,iVBORw0KGgo=" + elements: Final[list[GeminiEmbeddingElement]] = [ChatCompletionTextObject(type="text", text=media), media] + embedding_input: Final[GeminiEmbeddingInput] = [elements] if combined else elements + vectors: Final = [[0.1]] if combined else [[0.1], [0.2]] + route: Final = respx_mock.post(GEMINI_2_BATCH_EMBEDDINGS_URL).mock( + return_value=httpx.Response(200, json={"embeddings": [{"values": vector} for vector in vectors]}) + ) + + response: Final = await litellm.aembedding( + model="gemini/gemini-embedding-2", input=embedding_input, api_key="gemini-test-key" + ) + + parts: Final = [{"text": media}, {"inline_data": {"mime_type": "image/png", "data": "iVBORw0KGgo="}}] + groups: Final = [parts] if combined else [[part] for part in parts] + assert json.loads(route.calls[0].request.content) == { + "requests": [{"model": "models/gemini-embedding-2", "content": {"parts": group}} for group in groups] + } + assert [item["embedding"] for item in response.data] == vectors + assert [item["index"] for item in response.data] == list(range(len(vectors))) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_vertex_embedding_sends_media_prefixed_text_as_literal_parts( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, +) -> None: + from litellm.main import google_batch_embeddings + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + credentials_key: Final = "test-vertex-embedding-credentials" + monkeypatch.setitem( + google_batch_embeddings._credentials_project_mapping, + (credentials_key, "test-project"), + (Credentials(token="test-vertex-token"), "test-project"), + ) + url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/" + "locations/us-central1/publishers/google/models/gemini-embedding-2:embedContent" + ) + route: Final = respx_mock.post(url).mock( + return_value=httpx.Response(200, json={"embedding": {"values": [0.1, 0.2]}}) + ) + text: Final = "gs://my-bucket/build/output.txt" + embedding_input: Final = [ChatCompletionTextObject(type="text", text=text)] + + response: Final = ( + litellm.embedding( + model="vertex_ai/gemini-embedding-2", + input=embedding_input, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=credentials_key, + ) + if sync_mode + else await litellm.aembedding( + model="vertex_ai/gemini-embedding-2", + input=embedding_input, + vertex_project="test-project", + vertex_location="us-central1", + vertex_credentials=credentials_key, + ) + ) + + assert route.call_count == 1 + assert len(respx_mock.calls) == 1 + assert json.loads(route.calls[0].request.content) == {"content": {"parts": [{"text": text}]}} + assert [item["embedding"] for item in response.data] == [[0.1, 0.2]] + assert response.usage.prompt_tokens == litellm.token_counter(model="gemini-embedding-2", text=text) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "bad_input", + [ + ["data:application/octet-stream;base64,SGVsbG8=", "plain text chunk"], + ["gs://my-bucket/build/output.txt", "plain text chunk"], + ], +) +async def test_media_prefix_validation_fails_as_bad_request_without_sending_a_request( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + bad_input: list[str], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with pytest.raises(BadRequestError) as excinfo: + await litellm.aembedding(model="gemini/gemini-embedding-2", input=bad_input, api_key="gemini-test-key") + + assert excinfo.value.status_code == 400 + assert len(respx_mock.calls) == 0 diff --git a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index d7d921ab35c..2cfe05ffbb6 100644 --- a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -21,8 +21,10 @@ from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( _build_part_for_input, - file_reference_name, _is_multimodal_input, + _source_mime_type, + file_reference_name, + flatten_media_sources, process_embed_content_response, process_response, transform_openai_input_gemini_content, @@ -305,7 +307,8 @@ class TestProcessResponse: assert result.usage.prompt_tokens > 0 def test_nested_empty_list_raises(self): - with pytest.raises(ValueError, match="must not be empty"): + # local input validation: a 400 BadRequestError, not a ValueError that surfaces as a 500 + with pytest.raises(BadRequestError, match="must not be empty"): transform_openai_input_gemini_content( input=[[]], model="gemini-embedding-2-preview", @@ -313,7 +316,9 @@ class TestProcessResponse: ) def test_nested_non_string_element_raises(self): - with pytest.raises(BadRequestError, match="must be strings or file content blocks, got list"): + with pytest.raises( + BadRequestError, match="must be strings, text content blocks, or file content blocks, got list" + ): transform_openai_input_gemini_content( input=[[["doubly", "nested"]]], model="gemini-embedding-2-preview", @@ -519,7 +524,8 @@ class TestFileContentBlocks: assert parts[1] == {"text": "a solid color clip"} def test_embed_content_path_still_rejects_nested_lists(self) -> None: - with pytest.raises(ValueError, match="Nested"): + # local input validation: a 400 BadRequestError, not a ValueError that surfaces as a 500 + with pytest.raises(BadRequestError, match="Nested"): transform_openai_input_gemini_embed_content( input=[[_file_block(file_data=VIDEO_DATA_URI), "a solid color clip"]], model=self.MODEL, @@ -548,9 +554,7 @@ class TestFileContentBlocks: ({"type": "image_url", "image_url": {"url": IMAGE_DATA_URI}}, "Input should be 'file'"), ], ) - def test_malformed_block_answers_400_naming_the_field( - self, block: dict[str, object], named_in_error: str - ) -> None: + def test_malformed_block_answers_400_naming_the_field(self, block: dict[str, object], named_in_error: str) -> None: with pytest.raises(BadRequestError, match=named_in_error): _build_part_for_input(block) @@ -652,3 +656,178 @@ def test_gemini_embedding(respx_mock: respx.MockRouter) -> None: expected_prompt_tokens: Final = litellm.token_counter(model="gemini/gemini-embedding-001", text="Hello, world!") assert response.usage.prompt_tokens == expected_prompt_tokens assert response.usage.total_tokens == expected_prompt_tokens + + +TEXT_BLOCK_DATA_PREFIX: Final = { + "type": "text", + "text": "data:application/octet-stream;base64,SGVsbG8= and surrounding prose", +} +TEXT_BLOCK_GCS: Final = {"type": "text", "text": "gs://my-bucket/build/output.txt"} +TEXT_BLOCK_FILES: Final = {"type": "text", "text": "files/README.md is the docs entry point"} + + +def _text_block(**overrides: object) -> dict[str, object]: + block: Final = {"type": "text", "text": "some plain text"} + block.update(overrides) + return block + + +class TestTypedTextBlocks: + """A {"type": "text"} block is embedded verbatim: no media prefix detection runs on it. + + Regression for #46719: text chunks that happen to start with data:, gs:// or + files/ were read as media, with no way for the caller to say they are text. + """ + + def test_data_url_prefixed_text_is_embedded_as_text(self) -> None: + part: Final = _build_part_for_input(TEXT_BLOCK_DATA_PREFIX) + assert part == {"text": TEXT_BLOCK_DATA_PREFIX["text"]} + + def test_gcs_prefixed_text_is_embedded_as_text(self) -> None: + part: Final = _build_part_for_input(TEXT_BLOCK_GCS) + assert part == {"text": TEXT_BLOCK_GCS["text"]} + + def test_files_prefixed_text_is_embedded_as_text(self) -> None: + part: Final = _build_part_for_input(TEXT_BLOCK_FILES) + assert part == {"text": TEXT_BLOCK_FILES["text"]} + + def test_text_block_keeps_detection_for_other_elements(self) -> None: + parts: Final = [_build_part_for_input(element) for element in [TEXT_BLOCK_DATA_PREFIX, IMAGE_DATA_URI]] + assert parts == [ + {"text": TEXT_BLOCK_DATA_PREFIX["text"]}, + {"inline_data": {"mime_type": "image/png", "data": IMAGE_DATA_URI.split(",", 1)[1]}}, + ] + + def test_text_block_is_not_multimodal(self) -> None: + assert _is_multimodal_input([_text_block(text="data:image/png;base64,iVBORw0KGgo=")]) is False + + def test_mixed_input_with_text_block_is_multimodal(self) -> None: + assert _is_multimodal_input([_text_block(), IMAGE_DATA_URI]) is True + + def test_empty_text_block_embeds_like_an_empty_string(self) -> None: + assert _build_part_for_input(_text_block(text="")) == {"text": ""} + + def test_text_block_with_extra_keys_is_rejected(self) -> None: + with pytest.raises(BadRequestError): + _build_part_for_input(_text_block(cache_control={"type": "ephemeral"})) + + def test_text_block_with_extra_keys_is_dropped_under_drop_params(self) -> None: + part: Final = _build_part_for_input(_text_block(cache_control={"type": "ephemeral"}), drop_params=True) + assert part == {"text": "some plain text"} + + def test_text_block_missing_text_is_rejected(self) -> None: + with pytest.raises(BadRequestError): + _build_part_for_input({"type": "text"}) + + def test_text_block_non_string_text_is_rejected(self) -> None: + with pytest.raises(BadRequestError): + _build_part_for_input(_text_block(text=123)) + + def test_nested_group_with_text_block_builds_one_combined_request(self) -> None: + body: Final = transform_openai_input_gemini_content( + input=[[TEXT_BLOCK_GCS, "plain chunk"]], + model="gemini-embedding-001", + optional_params={}, + ) + assert body["requests"] == [ + { + "model": "models/gemini-embedding-001", + "content": {"parts": [{"text": TEXT_BLOCK_GCS["text"]}, {"text": "plain chunk"}]}, + } + ] + + +class TestTextBlocksAreNeverFileReferences: + def test_flatten_media_sources_skips_text_blocks(self) -> None: + sources: Final = flatten_media_sources([TEXT_BLOCK_FILES, "a red bus"]) + assert sources == ("a red bus",) + + +class TestLocalValidationIsClientError: + """Local input validation failures are BadRequestError (400), not ValueError-turned-500. + + No request was sent when they raise, so surfacing them as APIConnectionError + makes retry logic retry a permanent input error until its time limit (#46719). + """ + + def test_unsupported_data_url_mime_is_a_client_error(self) -> None: + with pytest.raises(BadRequestError) as excinfo: + transform_openai_input_gemini_content( + input=["data:application/octet-stream;base64,SGVsbG8="], + model="gemini-embedding-001", + optional_params={}, + ) + assert "Unsupported MIME type for embedding" in str(excinfo.value) + + def test_gcs_unresolvable_extension_is_a_client_error(self) -> None: + with pytest.raises(BadRequestError) as excinfo: + transform_openai_input_gemini_content( + input=["gs://my-bucket/build/output.txt"], + model="gemini-embedding-001", + optional_params={}, + ) + assert "Unable to infer MIME type from GCS URL" in str(excinfo.value) + + def test_gcs_mime_type_override_skips_inference(self) -> None: + part: Final = _build_part_for_input(_file_block(file_id="gs://my-bucket/build/output.txt", format="text/plain")) + assert part == {"file_data": {"mime_type": "text/plain", "file_uri": "gs://my-bucket/build/output.txt"}} + + def test_source_mime_type_soft_probe_still_returns_none(self) -> None: + assert _source_mime_type("data:application/octet-stream;base64,SGVsbG8=", None, {}) is None + assert _source_mime_type("gs://my-bucket/build/output.txt", None, {}) is None + + def test_empty_nested_group_is_a_client_error(self) -> None: + with pytest.raises(BadRequestError): + transform_openai_input_gemini_content( + input=[["a red bus"], []], + model="gemini-embedding-001", + optional_params={}, + ) + + def test_embed_content_nested_input_is_a_client_error(self) -> None: + with pytest.raises(BadRequestError): + transform_openai_input_gemini_embed_content( + input=[["a red bus", IMAGE_DATA_URI]], + model="gemini-embedding-001", + optional_params={}, + ) + + +class TestTextBlockTokenAccounting: + """A text block must count its text exactly like the same text as a bare string.""" + + def _predictions(self) -> VertexAIBatchEmbeddingsResponseObject: + return VertexAIBatchEmbeddingsResponseObject(embeddings=[{"values": [0.1, 0.2]}, {"values": [0.3, 0.4]}]) + + def test_text_block_counts_like_the_same_bare_string(self) -> None: + chunk: Final = "the quick brown fox starts a chunk" + via_block: Final = process_response( + input=[_text_block(text=chunk), IMAGE_DATA_URI], + model_response=EmbeddingResponse(), + model="gemini-embedding-001", + _predictions=self._predictions(), + ) + via_string: Final = process_response( + input=[chunk, IMAGE_DATA_URI], + model_response=EmbeddingResponse(), + model="gemini-embedding-001", + _predictions=self._predictions(), + ) + assert via_block.usage.prompt_tokens == via_string.usage.prompt_tokens + assert via_block.usage.prompt_tokens > 0 + + def test_embed_content_fallback_counts_text_block_like_a_bare_string(self) -> None: + text: Final = "hello world this is plain text" + via_block: Final = process_embed_content_response( + input=[_text_block(text=text)], + model_response=EmbeddingResponse(), + model="gemini-embedding-001", + response_json={"embedding": {"values": [0.1, 0.2]}}, + ) + via_string: Final = process_embed_content_response( + input=[text], + model_response=EmbeddingResponse(), + model="gemini-embedding-001", + response_json={"embedding": {"values": [0.1, 0.2]}}, + ) + assert via_block.usage.prompt_tokens == via_string.usage.prompt_tokens > 0