From fdf970b6d25cddc9ce19a2655980870c75e6d322 Mon Sep 17 00:00:00 2001 From: choksi2212 Date: Fri, 16 Oct 2026 00:42:23 +0530 Subject: [PATCH 1/3] fix(vertex_ai): embed typed text blocks verbatim in Gemini embeddings and fail local media-prefix validation as a 400 Gemini embeddings read any bare string that starts with data:, gs:// or files/ as media, with no way to send it as text. A source-code or document chunk that happens to start with one of those prefixes is silently embedded as the wrong modality, or fails local validation as an APIConnectionError (500), which retry logic retries until its time limit because no request was ever sent. - Accept {"type": "text", "text": ...} content blocks in the embeddings input, mirroring the existing file content blocks: the text is embedded verbatim, no media-prefix detection runs on it, and it is never resolved as a Gemini Files API reference. - Raise local input validation (unsupported data: MIME type, unresolvable gs:// extension, empty nested group, nested input on the embedContent path) as BadRequestError (400) instead of bare ValueError, which aembedding()/embedding() map to APIConnectionError (500). - Count text blocks like the same bare strings for token accounting, on the batchEmbedContents usage path and the embedContent fallback. Closes #46719 Signed-off-by: choksi2212 --- .../batch_embed_content_transformation.py | 131 ++++++++++-- litellm/types/llms/vertex_ai.py | 8 +- .../test_batch_embed_content_handler.py | 72 ++++++- ...test_batch_embed_content_transformation.py | 193 +++++++++++++++++- 4 files changed, 372 insertions(+), 32 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 96a785c06c9..e1cd25a197c 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -166,11 +166,20 @@ class _EmbeddingFileBlock(LiteLLMBaseModel): file: _EmbeddingFile +class _EmbeddingTextBlock(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + type: Literal["text"] + text: str + + _file_block_adapter: Final = TypeAdapter(_EmbeddingFileBlock) +_text_block_adapter: Final = TypeAdapter(_EmbeddingTextBlock) _video_metadata_adapter: Final = TypeAdapter(VideoMetadataType) _input_shape_adapter: Final[TypeAdapter[str | list[object]]] = TypeAdapter(str | list[object]) _mapping_adapter: Final = TypeAdapter(dict[str, object]) _BLOCK_FIELDS: Final = frozenset(_EmbeddingFileBlock.model_fields) +_TEXT_BLOCK_FIELDS: Final = frozenset(_EmbeddingTextBlock.model_fields) _FILE_FIELDS: Final = frozenset(_EmbeddingFile.model_fields) _VIDEO_METADATA_FIELDS: Final = frozenset(_EmbeddingVideoMetadata.model_fields) _FILE_SOURCE_FORMS: Final = "a data: URI, a gs:// URL, a files/ reference, or a Gemini Files API URI" @@ -211,10 +220,33 @@ def _dropping_unsupported_keys(block: Mapping[str, object]) -> dict[str, object] } +def _is_text_block(element: object) -> bool: + """A typed text content block: a mapping with type=text and a text string. + + A text block is embedded as text, verbatim: no media prefix detection + runs on it, no matter that its text starts with data:, gs:// or files/. + """ + return isinstance(element, Mapping) and element.get("type") == "text" + + +def _parse_text_block(element: object, drop_params: bool) -> _EmbeddingTextBlock: + if not isinstance(element, Mapping): + raise _invalid_input( + f"Embedding input elements must be strings, text content blocks, or file content blocks, got {type(element).__name__}" + ) + block: Final = cast(Mapping[str, object], element) # cast-ok: isinstance leaves the key and value types unknown + try: + return _text_block_adapter.validate_python(_only_fields(block, _TEXT_BLOCK_FIELDS) if drop_params else block) + except ValidationError as error: + raise _invalid_input( + f"Invalid text content block in embedding input: {_validation_error_summary(error)}" + ) from error + + def _parse_file_block(element: object, drop_params: bool) -> _EmbeddingFileBlock: if not isinstance(element, Mapping): raise _invalid_input( - f"Embedding input elements must be strings or file content blocks, got {type(element).__name__}" + f"Embedding input elements must be strings, text content blocks, or file content blocks, got {type(element).__name__}" ) block: Final = cast(Mapping[str, object], element) # cast-ok: isinstance leaves the key and value types unknown try: @@ -268,16 +300,33 @@ def _source_mime_type( return None +def _gcs_mime_type(gcs_url: str, mime_type_override: str | None) -> str: + """The MIME type for a GCS file part: an explicit override, else one inferred from the URL extension. + + An unresolvable extension is a client input error (400), not a connection + error (500): no request was sent when it is raised. + """ + if mime_type_override is not None: + return mime_type_override + try: + return _infer_mime_type_from_gcs_url(gcs_url) + except ValueError as error: + raise _invalid_input(str(error)) from error + + def _media_part( source: str, mime_type_override: str | None, resolved_files: Mapping[str, Mapping[str, str]], ) -> PartType: if _is_data_url(source): - mime_type, base64_data = _parse_data_url(source, mime_type_override) - return PartType(inline_data=BlobType(mime_type=mime_type, data=base64_data)) + try: + data_url_parts: Final = _parse_data_url(source, mime_type_override) + except ValueError as error: + raise _invalid_input(str(error)) from error + return PartType(inline_data=BlobType(mime_type=data_url_parts[0], data=data_url_parts[1])) if _is_gcs_url(source): - gcs_mime_type: Final = mime_type_override or _infer_mime_type_from_gcs_url(source) + gcs_mime_type: Final = _gcs_mime_type(source, mime_type_override) return PartType(file_data=FileDataType(mime_type=gcs_mime_type, file_uri=source)) if is_file_reference(source): file_info: Final = resolved_files.get(source) @@ -299,7 +348,7 @@ def _top_level_elements( _input_shape_adapter.validate_python(input) except ValidationError as error: raise _invalid_input( - f"Embedding input must be a string or a list of strings and file content blocks, got {type(input).__name__}" + f"Embedding input must be a string or a list of strings, text content blocks, and file content blocks, got {type(input).__name__}" ) from error return [input] if isinstance(input, str) else input @@ -312,15 +361,25 @@ def _elements(input: GeminiEmbeddingInput) -> Iterator[GeminiEmbeddingElement]: yield element -def _element_source(element: GeminiEmbeddingElement) -> str: +def _element_source(element: GeminiEmbeddingElement) -> str | None: + """The media source of an element: the string itself, or a file content block's source. + + Typed text blocks are not media sources: their text is declared text, so it + must never be resolved as a file reference or parsed as an inline/GCS media part. + """ if isinstance(element, str): return element + if _is_text_block(element): + return None return _file_block_source(_parse_file_block(element, drop_params=True)) def flatten_media_sources(input: GeminiEmbeddingInput) -> tuple[str, ...]: - """Every string element plus every file content block's source, in input order.""" - return tuple(_element_source(element) for element in _elements(input)) + """Every string element plus every file content block's source, in input order. + + Typed text blocks contribute nothing: their text is declared text, never media. + """ + return tuple(source for source in (_element_source(element) for element in _elements(input)) if source is not None) def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool: @@ -338,9 +397,9 @@ def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool: def _is_multimodal_element(element: GeminiEmbeddingElement) -> bool: - """Check if a single element is multimodal.""" + """Check if a single element is multimodal. Typed text blocks are text, not media.""" if not isinstance(element, str): - return True + return not _is_text_block(element) if _is_data_url(element): return True if is_file_reference(element): @@ -365,6 +424,9 @@ def _build_part_for_input( if isinstance(element, str): return _media_part(element, None, files) if _is_multimodal_element(element) else PartType(text=element) + if _is_text_block(element): + return PartType(text=_parse_text_block(element, drop_params).text) + block: Final = _parse_file_block(element, drop_params) part: Final = _media_part(_file_block_source(block), block.file.format, files) if block.file.video_metadata is None: @@ -400,7 +462,7 @@ def transform_openai_input_gemini_content( Transform OpenAI embedding input to Gemini batchEmbedContents format. Each input element becomes a separate EmbedContentRequest, supporting - text, data URIs, file references, and GCS URLs. + text (bare strings and typed text blocks), data URIs, file references, and GCS URLs. If an element is a list (nested input), all sub-elements are combined into a single content with multiple parts, producing one combined @@ -421,7 +483,7 @@ def transform_openai_input_gemini_content( for element in input_list: if isinstance(element, list): if not element: - raise ValueError("Nested input list must not be empty") + raise _invalid_input("Nested input list must not be empty") parts = [ _build_part_for_input(sub, resolved_files=resolved_files, drop_params=drop_params) for sub in element ] @@ -448,7 +510,7 @@ def transform_openai_input_gemini_embed_content( Transform OpenAI embedding input to Gemini embedContent format (multimodal). Args: - input: GeminiEmbeddingInput with text, data URIs, or file references + input: GeminiEmbeddingInput with text (bare strings and typed text blocks), data URIs, or file references model: Model name optional_params: Additional parameters (taskType, outputDimensionality, etc.) resolved_files: Dict mapping file names (files/abc) to {mime_type, uri} @@ -465,7 +527,7 @@ def transform_openai_input_gemini_embed_content( for element in input_list: if isinstance(element, list): - raise ValueError( + raise _invalid_input( "Nested (combined) embeddings are not supported on the embedContent path. " "Use the batchEmbedContents path or pass a flat list instead." ) @@ -502,6 +564,8 @@ def _is_image_element( ) -> bool: if isinstance(element, str): return _source_mime_type(element, None, resolved_files) in _IMAGE_MIME_TYPES + if _is_text_block(element): + return False block: Final = _parse_file_block(element, drop_params=True) return _source_mime_type(_file_block_source(block), block.file.format, resolved_files) in _IMAGE_MIME_TYPES @@ -514,6 +578,33 @@ def _is_image_only_input( return bool(elements) and all(_is_image_element(element, resolved_files) for element in elements) +def _plain_text(element: GeminiEmbeddingElement) -> str | None: + """The text of an element that is text, not media: a plain string, or a typed text block.""" + if isinstance(element, str): + return None if _is_multimodal_element(element) else element + if _is_text_block(element): + return _parse_text_block(element, drop_params=True).text + return None + + +def _text_substituted_input(input: GeminiEmbeddingInput) -> GeminiEmbeddingInput: + """Input with typed text blocks replaced by their text, so token counting sees plain strings. + + Inputs without text blocks keep their shape and contents unchanged. + """ + + def _substitute(element: object) -> object: + if isinstance(element, list): + return [_substitute(sub) for sub in element] + if _is_text_block(element): + return _parse_text_block(element, drop_params=True).text + return element + + if not isinstance(input, list): + return input + return [_substitute(element) for element in input] + + def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int: return sum(detail["tokenCount"] for detail in details if detail["modality"] == modality) @@ -521,7 +612,7 @@ def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) def _fallback_usage(input: GeminiEmbeddingInput, model: str) -> Usage: if _is_multimodal_input(input): return Usage(prompt_tokens=0, total_tokens=0) - input_text: Final = get_formatted_prompt(data={"input": input}, call_type="embedding") + input_text: Final = get_formatted_prompt(data={"input": _text_substituted_input(input)}, call_type="embedding") prompt_tokens: Final = token_counter(model=model, text=input_text) return Usage(prompt_tokens=prompt_tokens, total_tokens=prompt_tokens) @@ -634,16 +725,18 @@ def process_response( text_elements: Final[list[str]] = [] for e in input_list: if isinstance(e, list): - text_elements.extend(sub for sub in e if isinstance(sub, str) and not _is_multimodal_element(sub)) - elif isinstance(e, str) and not _is_multimodal_element(e): - text_elements.append(e) + text_elements.extend(text for text in (_plain_text(sub) for sub in e) if text is not None) + else: + plain: str | None = _plain_text(e) + if plain is not None: + text_elements.append(plain) if text_elements: input_text = get_formatted_prompt(data={"input": text_elements}, call_type="embedding") prompt_tokens = token_counter(model=model, text=input_text) else: prompt_tokens = 0 else: - input_text = get_formatted_prompt(data={"input": input}, call_type="embedding") + input_text = get_formatted_prompt(data={"input": _text_substituted_input(input)}, call_type="embedding") prompt_tokens = token_counter(model=model, text=input_text) model_response.usage = Usage(prompt_tokens=prompt_tokens, total_tokens=prompt_tokens) diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 116fdda0b5d..c6ad6ec283c 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -7,12 +7,14 @@ from typing_extensions import ( TypedDict, ) -from litellm.types.llms.openai import ChatCompletionFileObject, EmbeddingInput +from litellm.types.llms.openai import ChatCompletionFileObject, ChatCompletionTextObject, EmbeddingInput # Gemini supports nested-list inputs (e.g. [["text", "image"]]) as an explicit -# opt-in for combined embeddings — a provider-specific extension of the +# opt-in for combined embeddings, and typed content blocks: a {"type": "text", +# "text": ...} block embeds its text verbatim with no media-prefix detection, +# while file blocks carry media — provider-specific extensions of the # OpenAI-faithful EmbeddingInput shape. -GeminiEmbeddingElement = str | ChatCompletionFileObject +GeminiEmbeddingElement = str | ChatCompletionFileObject | ChatCompletionTextObject GeminiEmbeddingInput = ( EmbeddingInput | list[GeminiEmbeddingElement] | list[list[str]] | list[list[GeminiEmbeddingElement]] ) diff --git a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py index 231bc25a7f0..f75965a43e3 100644 --- a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py +++ b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py @@ -6,6 +6,7 @@ import pytest import respx import litellm +from litellm.exceptions import BadRequestError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler import GoogleBatchEmbeddings @@ -67,9 +68,7 @@ async def test_gemini_embedding_sends_batch_request_and_parses_vectors( ] } assert [item["embedding"] for item in response.data] == [[0.1, 0.2, 0.3]] - local_token_count: Final = litellm.token_counter( - model="gemini-embedding-001", text="good morning from litellm" - ) + local_token_count: Final = litellm.token_counter(model="gemini-embedding-001", text="good morning from litellm") assert local_token_count > 0 assert (response.usage.prompt_tokens, response.usage.total_tokens) == ( local_token_count, @@ -88,3 +87,70 @@ async def test_async_resolve_file_references_fetches_the_file_metadata_for_both_ async_handler=AsyncHTTPHandler(transport=httpx.MockTransport(_files_api_metadata)), ) assert resolved == {reference: {"mime_type": "video/mp4", "uri": FILES_URI}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_files_prefixed_text_block_embeds_as_text_without_a_files_api_lookup( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, +) -> None: + """Regression for #46719: a typed text block whose text starts with files/ is text, not a file reference. + + No Files API GET may fire for it, and the batch request must carry the text verbatim. + """ + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post(GEMINI_BATCH_EMBEDDINGS_URL).mock( + return_value=httpx.Response(200, json={"embeddings": [{"values": [0.1, 0.2, 0.3]}]}) + ) + + embedding_input: Final = [{"type": "text", "text": "files/README.md is the docs entry point"}] + response: Final = ( + litellm.embedding(model="gemini/gemini-embedding-001", input=embedding_input, api_key="gemini-test-key") + if sync_mode + else await litellm.aembedding( + model="gemini/gemini-embedding-001", input=embedding_input, api_key="gemini-test-key" + ) + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == { + "requests": [ + { + "model": "models/gemini-embedding-001", + "content": {"parts": [{"text": "files/README.md is the docs entry point"}]}, + } + ] + } + assert [item["embedding"] for item in response.data] == [[0.1, 0.2, 0.3]] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "bad_input", + [ + ["data:application/octet-stream;base64,SGVsbG8=", "plain text chunk"], + ["gs://my-bucket/build/output.txt", "plain text chunk"], + ], +) +async def test_media_prefix_validation_fails_as_bad_request_without_sending_a_request( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + bad_input: list[str], +) -> None: + """Regression for #46719: local input validation is a 400 BadRequestError, not a 500 connection error. + + These are client input errors raised before any request: retry logic must not + retry them until its time limit, and no HTTP call may be made. + """ + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post(GEMINI_BATCH_EMBEDDINGS_URL).mock( + return_value=httpx.Response(200, json={"embeddings": [{"values": [0.1, 0.2, 0.3]}]}) + ) + + with pytest.raises(BadRequestError) as excinfo: + await litellm.aembedding(model="gemini/gemini-embedding-001", input=bad_input, api_key="gemini-test-key") + + assert excinfo.value.status_code == 400 + assert route.call_count == 0 diff --git a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index d7d921ab35c..6e25061a8a2 100644 --- a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -21,8 +21,10 @@ from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( _build_part_for_input, - file_reference_name, _is_multimodal_input, + _source_mime_type, + file_reference_name, + flatten_media_sources, process_embed_content_response, process_response, transform_openai_input_gemini_content, @@ -305,7 +307,8 @@ class TestProcessResponse: assert result.usage.prompt_tokens > 0 def test_nested_empty_list_raises(self): - with pytest.raises(ValueError, match="must not be empty"): + # local input validation: a 400 BadRequestError, not a ValueError that surfaces as a 500 + with pytest.raises(BadRequestError, match="must not be empty"): transform_openai_input_gemini_content( input=[[]], model="gemini-embedding-2-preview", @@ -313,7 +316,9 @@ class TestProcessResponse: ) def test_nested_non_string_element_raises(self): - with pytest.raises(BadRequestError, match="must be strings or file content blocks, got list"): + with pytest.raises( + BadRequestError, match="must be strings, text content blocks, or file content blocks, got list" + ): transform_openai_input_gemini_content( input=[[["doubly", "nested"]]], model="gemini-embedding-2-preview", @@ -519,7 +524,8 @@ class TestFileContentBlocks: assert parts[1] == {"text": "a solid color clip"} def test_embed_content_path_still_rejects_nested_lists(self) -> None: - with pytest.raises(ValueError, match="Nested"): + # local input validation: a 400 BadRequestError, not a ValueError that surfaces as a 500 + with pytest.raises(BadRequestError, match="Nested"): transform_openai_input_gemini_embed_content( input=[[_file_block(file_data=VIDEO_DATA_URI), "a solid color clip"]], model=self.MODEL, @@ -548,9 +554,7 @@ class TestFileContentBlocks: ({"type": "image_url", "image_url": {"url": IMAGE_DATA_URI}}, "Input should be 'file'"), ], ) - def test_malformed_block_answers_400_naming_the_field( - self, block: dict[str, object], named_in_error: str - ) -> None: + def test_malformed_block_answers_400_naming_the_field(self, block: dict[str, object], named_in_error: str) -> None: with pytest.raises(BadRequestError, match=named_in_error): _build_part_for_input(block) @@ -652,3 +656,178 @@ def test_gemini_embedding(respx_mock: respx.MockRouter) -> None: expected_prompt_tokens: Final = litellm.token_counter(model="gemini/gemini-embedding-001", text="Hello, world!") assert response.usage.prompt_tokens == expected_prompt_tokens assert response.usage.total_tokens == expected_prompt_tokens + + +TEXT_BLOCK_DATA_PREFIX: Final = { + "type": "text", + "text": "data:application/octet-stream;base64,SGVsbG8= and surrounding prose", +} +TEXT_BLOCK_GCS: Final = {"type": "text", "text": "gs://my-bucket/build/output.txt"} +TEXT_BLOCK_FILES: Final = {"type": "text", "text": "files/README.md is the docs entry point"} + + +def _text_block(**overrides: object) -> dict[str, object]: + block: Final = {"type": "text", "text": "some plain text"} + block.update(overrides) + return block + + +class TestTypedTextBlocks: + """A {"type": "text"} block is embedded verbatim: no media prefix detection runs on it. + + Regression for #46719: text chunks that happen to start with data:, gs:// or + files/ were read as media, with no way for the caller to say they are text. + """ + + def test_data_url_prefixed_text_is_embedded_as_text(self) -> None: + part: Final = _build_part_for_input(TEXT_BLOCK_DATA_PREFIX) + assert part == {"text": TEXT_BLOCK_DATA_PREFIX["text"]} + + def test_gcs_prefixed_text_is_embedded_as_text(self) -> None: + part: Final = _build_part_for_input(TEXT_BLOCK_GCS) + assert part == {"text": TEXT_BLOCK_GCS["text"]} + + def test_files_prefixed_text_is_embedded_as_text(self) -> None: + part: Final = _build_part_for_input(TEXT_BLOCK_FILES) + assert part == {"text": TEXT_BLOCK_FILES["text"]} + + def test_text_block_keeps_detection_for_other_elements(self) -> None: + parts: Final = [_build_part_for_input(element) for element in [TEXT_BLOCK_DATA_PREFIX, IMAGE_DATA_URI]] + assert parts == [ + {"text": TEXT_BLOCK_DATA_PREFIX["text"]}, + {"inline_data": {"mime_type": "image/png", "data": IMAGE_DATA_URI.split(",", 1)[1]}}, + ] + + def test_text_block_is_not_multimodal(self) -> None: + assert _is_multimodal_input([_text_block(text="data:image/png;base64,iVBORw0KGgo=")]) is False + + def test_mixed_input_with_text_block_is_multimodal(self) -> None: + assert _is_multimodal_input([_text_block(), IMAGE_DATA_URI]) is True + + def test_empty_text_block_embeds_like_an_empty_string(self) -> None: + assert _build_part_for_input(_text_block(text="")) == {"text": ""} + + def test_text_block_with_extra_keys_is_rejected(self) -> None: + with pytest.raises(BadRequestError): + _build_part_for_input(_text_block(cache_control={"type": "ephemeral"})) + + def test_text_block_with_extra_keys_is_dropped_under_drop_params(self) -> None: + part: Final = _build_part_for_input(_text_block(cache_control={"type": "ephemeral"}), drop_params=True) + assert part == {"text": "some plain text"} + + def test_text_block_missing_text_is_rejected(self) -> None: + with pytest.raises(BadRequestError): + _build_part_for_input({"type": "text"}) + + def test_text_block_non_string_text_is_rejected(self) -> None: + with pytest.raises(BadRequestError): + _build_part_for_input(_text_block(text=123)) + + def test_nested_group_with_text_block_builds_one_combined_request(self) -> None: + body: Final = transform_openai_input_gemini_content( + input=[[TEXT_BLOCK_GCS, "plain chunk"]], + model="gemini-embedding-001", + optional_params={}, + ) + assert body["requests"] == [ + { + "model": "models/gemini-embedding-001", + "content": {"parts": [{"text": TEXT_BLOCK_GCS["text"]}, {"text": "plain chunk"}]}, + } + ] + + +class TestTextBlocksAreNeverFileReferences: + def test_flatten_media_sources_skips_text_blocks(self) -> None: + sources: Final = flatten_media_sources([TEXT_BLOCK_FILES, "a red bus"]) + assert sources == ("a red bus",) + + +class TestLocalValidationIsClientError: + """Local input validation failures are BadRequestError (400), not ValueError-turned-500. + + No request was sent when they raise, so surfacing them as APIConnectionError + makes retry logic retry a permanent input error until its time limit (#46719). + """ + + def test_unsupported_data_url_mime_is_a_client_error(self) -> None: + with pytest.raises(BadRequestError) as excinfo: + transform_openai_input_gemini_content( + input=["data:application/octet-stream;base64,SGVsbG8="], + model="gemini-embedding-001", + optional_params={}, + ) + assert "Unsupported MIME type for embedding" in str(excinfo.value) + + def test_gcs_unresolvable_extension_is_a_client_error(self) -> None: + with pytest.raises(BadRequestError) as excinfo: + transform_openai_input_gemini_content( + input=["gs://my-bucket/build/output.txt"], + model="gemini-embedding-001", + optional_params={}, + ) + assert "Unable to infer MIME type from GCS URL" in str(excinfo.value) + + def test_gcs_mime_type_override_skips_inference(self) -> None: + part: Final = _build_part_for_input(_file_block(file_id="gs://my-bucket/build/output.txt", format="text/plain")) + assert part == {"file_data": {"mime_type": "text/plain", "file_uri": "gs://my-bucket/build/output.txt"}} + + def test_source_mime_type_soft_probe_still_returns_none(self) -> None: + assert _source_mime_type("data:application/octet-stream;base64,SGVsbG8=", None, {}) is None + assert _source_mime_type("gs://my-bucket/build/output.txt", None, {}) is None + + def test_empty_nested_group_is_a_client_error(self) -> None: + with pytest.raises(BadRequestError): + transform_openai_input_gemini_content( + input=[["a red bus"], []], + model="gemini-embedding-001", + optional_params={}, + ) + + def test_embed_content_nested_input_is_a_client_error(self) -> None: + with pytest.raises(BadRequestError): + transform_openai_input_gemini_embed_content( + input=[["a red bus", IMAGE_DATA_URI]], + model="gemini-embedding-001", + optional_params={}, + ) + + +class TestTextBlockTokenAccounting: + """A text block must count its text exactly like the same text as a bare string.""" + + def _predictions(self) -> VertexAIBatchEmbeddingsResponseObject: + return VertexAIBatchEmbeddingsResponseObject(embeddings=[{"values": [0.1, 0.2]}]) + + def test_text_block_counts_like_the_same_bare_string(self) -> None: + chunk: Final = "the quick brown fox starts a chunk" + via_block: Final = process_response( + input=[_text_block(text=chunk), IMAGE_DATA_URI], + model_response=EmbeddingResponse(), + model="gemini-embedding-001", + _predictions=self._predictions(), + ) + via_string: Final = process_response( + input=[chunk, IMAGE_DATA_URI], + model_response=EmbeddingResponse(), + model="gemini-embedding-001", + _predictions=self._predictions(), + ) + assert via_block.usage.prompt_tokens == via_string.usage.prompt_tokens + assert via_block.usage.prompt_tokens > 0 + + def test_embed_content_fallback_counts_text_block_like_a_bare_string(self) -> None: + text: Final = "hello world this is plain text" + via_block: Final = process_embed_content_response( + input=[_text_block(text=text)], + model_response=EmbeddingResponse(), + model="gemini-embedding-001", + response_json={"embedding": {"values": [0.1, 0.2]}}, + ) + via_string: Final = process_embed_content_response( + input=[text], + model_response=EmbeddingResponse(), + model="gemini-embedding-001", + response_json={"embedding": {"values": [0.1, 0.2]}}, + ) + assert via_block.usage.prompt_tokens == via_string.usage.prompt_tokens > 0 From e53b195c1a7a322be377adbb5a7c0fe157984c80 Mon Sep 17 00:00:00 2001 From: choksi2212 Date: Fri, 16 Oct 2026 02:22:59 +0530 Subject: [PATCH 2/3] fix(vertex_ai): make the text-block substitution walk iterative and type-neutral CI flagged two quality gates on the previous revision of this change: - recursive_detector: _substitute was a recursive helper and the repo policy rejects new recursive functions. The nested-input walk now uses an explicit (source, sink) stack instead: same shape-preserving, order-preserving substitution, no recursion. - basedpyright delta gate: the walk introduced new reportReturnType / reportUnknown{Argument,Member,Variable}Type diagnostics. The stack locals are now fully annotated and the isinstance-narrowed containers are widened with the file's established cast-ok pattern, so the file contributes exactly zero new diagnostics vs the merge-base (verified with basedpyright 1.39.7 over the whole litellm tree: base and head produce identical per-rule totals). No behavior change: the full gemini_embeddings suite (105 tests) passes and ruff check/format stay clean. Signed-off-by: choksi2212 --- .../batch_embed_content_transformation.py | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index e1cd25a197c..0c0968d277f 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -226,7 +226,9 @@ def _is_text_block(element: object) -> bool: A text block is embedded as text, verbatim: no media prefix detection runs on it, no matter that its text starts with data:, gs:// or files/. """ - return isinstance(element, Mapping) and element.get("type") == "text" + return ( + isinstance(element, Mapping) and cast(Mapping[str, object], element).get("type") == "text" + ) # cast-ok: isinstance leaves the Mapping key/value types unknown def _parse_text_block(element: object, drop_params: bool) -> _EmbeddingTextBlock: @@ -590,19 +592,37 @@ def _plain_text(element: GeminiEmbeddingElement) -> str | None: def _text_substituted_input(input: GeminiEmbeddingInput) -> GeminiEmbeddingInput: """Input with typed text blocks replaced by their text, so token counting sees plain strings. - Inputs without text blocks keep their shape and contents unchanged. + Inputs without text blocks keep their shape and contents unchanged. The + walk is iterative on purpose (an explicit stack instead of a recursive + helper) to stay within the repo's recursive-function policy. """ - - def _substitute(element: object) -> object: - if isinstance(element, list): - return [_substitute(sub) for sub in element] - if _is_text_block(element): - return _parse_text_block(element, drop_params=True).text - return element - if not isinstance(input, list): return input - return [_substitute(element) for element in input] + + result: list[object] = [] + stack: list[tuple[list[object], list[object]]] = [ + (list(cast(list[object], input)), result) + ] # cast-ok: read-only widening; the walker never writes into input + while stack: + source, sink = stack[-1] + if not source: + stack.pop() + if stack: + sink_parent: list[object] = stack[-1][1] + sink_parent.append(sink) + continue + item: object = source.pop(0) + if isinstance(item, list): + stack.append( + (list(cast(list[object], item)), []) + ) # cast-ok: isinstance leaves the list element type unknown + elif _is_text_block(item): + sink.append(_parse_text_block(item, drop_params=True).text) + else: + sink.append(item) + # Shape-preserving substitution: a valid GeminiEmbeddingInput with its text + # blocks read as their text is still a valid GeminiEmbeddingInput. + return cast(GeminiEmbeddingInput, result) def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int: From 61ee75c34097b2ef619705c947029b3a41656f04 Mon Sep 17 00:00:00 2001 From: choksi2212 Date: Fri, 16 Oct 2026 04:22:13 +0530 Subject: [PATCH 3/3] fix(vertex_ai): keep type-discipline suppressions on the lines they cover The LIT gate anchors every *-ok suppression to the line its violation is reported on. The walker's cast-ok markers sat on the closing-paren line of their multi-line expressions, so each one both failed to suppress its cast (LIT006) and fired LIT013 for suppressing nothing. Move them onto the cast lines, give the return cast its own marker, declare the walker state Final with mutable-ok reasons for the annotations the walk genuinely mutates, inline the sink_parent read, and mark the two Gemini embedding aliases as TypeAlias. Signed-off-by: choksi2212 --- .../batch_embed_content_transformation.py | 22 +++++++++---------- litellm/types/llms/vertex_ai.py | 6 ++--- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 0c0968d277f..b8a62b86777 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -227,8 +227,9 @@ def _is_text_block(element: object) -> bool: runs on it, no matter that its text starts with data:, gs:// or files/. """ return ( - isinstance(element, Mapping) and cast(Mapping[str, object], element).get("type") == "text" - ) # cast-ok: isinstance leaves the Mapping key/value types unknown + isinstance(element, Mapping) + and cast(Mapping[str, object], element).get("type") == "text" # cast-ok: isinstance leaves value type unknown + ) def _parse_text_block(element: object, drop_params: bool) -> _EmbeddingTextBlock: @@ -599,30 +600,29 @@ def _text_substituted_input(input: GeminiEmbeddingInput) -> GeminiEmbeddingInput if not isinstance(input, list): return input - result: list[object] = [] - stack: list[tuple[list[object], list[object]]] = [ - (list(cast(list[object], input)), result) - ] # cast-ok: read-only widening; the walker never writes into input + result: Final[list[object]] = [] # mutable-ok: the walker accumulates into it; the name is never rebound + stack: Final[list[tuple[list[object], list[object]]]] = [ # mutable-ok: pop/append walker state; never rebound + (list(cast(list[object], input)), result) # cast-ok: read-only widening; the walker never writes into input + ] while stack: source, sink = stack[-1] if not source: stack.pop() if stack: - sink_parent: list[object] = stack[-1][1] - sink_parent.append(sink) + stack[-1][1].append(sink) continue item: object = source.pop(0) if isinstance(item, list): stack.append( - (list(cast(list[object], item)), []) - ) # cast-ok: isinstance leaves the list element type unknown + (list(cast(list[object], item)), []) # cast-ok: isinstance leaves the list element type unknown + ) elif _is_text_block(item): sink.append(_parse_text_block(item, drop_params=True).text) else: sink.append(item) # Shape-preserving substitution: a valid GeminiEmbeddingInput with its text # blocks read as their text is still a valid GeminiEmbeddingInput. - return cast(GeminiEmbeddingInput, result) + return cast(GeminiEmbeddingInput, result) # cast-ok: shape-preserving substitution; see the note above def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int: diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index c6ad6ec283c..030c8f24a8a 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Final, Literal, Protocol +from typing import Any, Final, Literal, Protocol, TypeAlias from typing_extensions import ( ReadOnly, @@ -14,8 +14,8 @@ from litellm.types.llms.openai import ChatCompletionFileObject, ChatCompletionTe # "text": ...} block embeds its text verbatim with no media-prefix detection, # while file blocks carry media — provider-specific extensions of the # OpenAI-faithful EmbeddingInput shape. -GeminiEmbeddingElement = str | ChatCompletionFileObject | ChatCompletionTextObject -GeminiEmbeddingInput = ( +GeminiEmbeddingElement: TypeAlias = str | ChatCompletionFileObject | ChatCompletionTextObject +GeminiEmbeddingInput: TypeAlias = ( EmbeddingInput | list[GeminiEmbeddingElement] | list[list[str]] | list[list[GeminiEmbeddingElement]] )