From d4f2119b03faa175e790dd86cb3c8aa46f546293 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 00:41:46 +0000 Subject: [PATCH 01/22] fix(cost): bill gemini-embedding-2 per token and stop double charging audio Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/llm_cost_calc/utils.py | 14 ++++- .../batch_embed_content_transformation.py | 60 ++----------------- ...odel_prices_and_context_window_backup.json | 15 ++--- model_prices_and_context_window.json | 15 ++--- .../llm_cost_calc/test_llm_cost_calc_utils.py | 33 ++++++++++ ...test_batch_embed_content_transformation.py | 42 +++++-------- 6 files changed, 75 insertions(+), 104 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 8fc428b38ae..a004f46b291 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -956,12 +956,17 @@ def _calculate_input_cost( ) ### AUDIO COST - if prompt_tokens_details["audio_tokens"]: + if prompt_tokens_details["audio_tokens"] and not ( + prompt_tokens_details["audio_length_seconds"] + and model_info.get("input_cost_per_audio_per_second") is not None + ): audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier) prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"]) ### IMAGE TOKEN COST - if prompt_tokens_details["image_tokens"]: + if prompt_tokens_details["image_tokens"] and not ( + prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None + ): # For image token costs: # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token. image_token_cost_key = "input_cost_per_image_token" @@ -970,7 +975,10 @@ def _calculate_input_cost( prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"]) ### VIDEO TOKEN COST - if prompt_tokens_details["video_tokens"]: + if prompt_tokens_details["video_tokens"] and not ( + prompt_tokens_details["video_length_seconds"] + and model_info.get("input_cost_per_video_per_second") is not None + ): video_token_cost_key = "input_cost_per_video_token" if model_info.get(video_token_cost_key) is None: video_token_cost_key = "input_cost_per_token" diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index e7fd9a0d08b..8e120ab9fe6 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -297,9 +297,6 @@ def transform_openai_input_gemini_embed_content( return request_body -_IMAGE_MIME_TYPES: Final = frozenset({"image/png", "image/jpeg"}) -_VIDEO_TOKENS_PER_SECOND: Final = 258.0 -_AUDIO_TOKENS_PER_SECOND: Final = 32.0 _usage_metadata_adapter: Final = TypeAdapter(UsageMetadata) @@ -312,40 +309,6 @@ def _parse_usage_metadata(raw_usage_metadata: object) -> UsageMetadata | None: return None -def _flatten_input(input: GeminiEmbeddingInput) -> tuple[str, ...]: - if isinstance(input, str): - return (input,) - return tuple(sub for element in input for sub in (element if isinstance(element, list) else [element])) - - -def _is_image_element( - element: str, - resolved_files: Mapping[str, Mapping[str, str]], -) -> bool: - if element.startswith("data:") and ";base64," in element: - try: - mime_type, _ = _parse_data_url(element) - except ValueError: - return False - return mime_type in _IMAGE_MIME_TYPES - if _is_gcs_url(element): - try: - return _infer_mime_type_from_gcs_url(element) in _IMAGE_MIME_TYPES - except ValueError: - return False - if _is_file_reference(element): - file_info: Final = resolved_files.get(element) - return file_info is not None and file_info.get("mime_type") in _IMAGE_MIME_TYPES - return False - - -def _count_input_images( - input: GeminiEmbeddingInput, - resolved_files: Mapping[str, Mapping[str, str]], -) -> int: - return sum(1 for element in _flatten_input(input) if _is_image_element(element, resolved_files)) - - def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int: return sum(detail["tokenCount"] for detail in details if detail["modality"] == modality) @@ -362,7 +325,6 @@ def _usage_from_embed_content_response( input: GeminiEmbeddingInput, model: str, raw_usage_metadata: object, - resolved_files: Mapping[str, Mapping[str, str]], ) -> Usage: usage_metadata: Final = _parse_usage_metadata(raw_usage_metadata) if usage_metadata is None: @@ -374,28 +336,17 @@ def _usage_from_embed_content_response( details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or () text_tokens: Final = _tokens_for_modality(details, "TEXT") audio_tokens: Final = _tokens_for_modality(details, "AUDIO") + image_tokens: Final = _tokens_for_modality(details, "IMAGE") video_tokens: Final = _tokens_for_modality(details, "VIDEO") - image_count: Final = _count_input_images(input, resolved_files) - - video_length_seconds: Final = video_tokens / _VIDEO_TOKENS_PER_SECOND if video_tokens > 0 else 0.0 - audio_length_seconds: Final = audio_tokens / _AUDIO_TOKENS_PER_SECOND if audio_tokens > 0 else 0.0 - - # generic_cost_per_token rewrites text_tokens to the full prompt minus - # other modalities when both text_tokens and image_count are zero. For - # video, that misallocates video tokens to text; a 1-token floor sidesteps - # the rewrite and keeps billing on input_cost_per_video_per_second. - needs_video_text_floor: Final = video_length_seconds > 0 and text_tokens == 0 and image_count == 0 - resolved_text_tokens: Final = 1 if needs_video_text_floor else text_tokens return Usage( prompt_tokens=prompt_tokens, total_tokens=total_tokens, prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=resolved_text_tokens, + text_tokens=text_tokens, audio_tokens=audio_tokens, - image_count=image_count, - video_length_seconds=video_length_seconds, - audio_length_seconds=audio_length_seconds, + image_tokens=image_tokens, + video_tokens=video_tokens, ), ) @@ -415,8 +366,6 @@ def process_embed_content_response( model_response: EmbeddingResponse to populate model: Model name response_json: Raw JSON response from embedContent endpoint - resolved_files: Mapping of file references (files/abc) to {mime_type, uri}, - used to bill resolved image references at the per-image rate Returns: EmbeddingResponse with single embedding @@ -438,7 +387,6 @@ def process_embed_content_response( input=input, model=model, raw_usage_metadata=response_json.get("usageMetadata"), - resolved_files=resolved_files or {}, ) return model_response diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0c60ff26635..74ca23fcc8a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25615,13 +25615,11 @@ "uses_embed_content": true }, "gemini-embedding-2": { - "input_cost_per_audio_per_second": 0.00016, "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_image": 0.00012, "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25648,13 +25646,11 @@ "uses_embed_content": true }, "vertex_ai/gemini-embedding-2": { - "input_cost_per_audio_per_second": 0.00016, "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_image": 0.00012, "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25709,10 +25705,11 @@ "tpm": 10000000 }, "gemini/gemini-embedding-2": { - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, + "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_token_batches": 1e-07, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0c60ff26635..74ca23fcc8a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25615,13 +25615,11 @@ "uses_embed_content": true }, "gemini-embedding-2": { - "input_cost_per_audio_per_second": 0.00016, "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_image": 0.00012, "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25648,13 +25646,11 @@ "uses_embed_content": true }, "vertex_ai/gemini-embedding-2": { - "input_cost_per_audio_per_second": 0.00016, "input_cost_per_audio_token": 6.5e-06, - "input_cost_per_image": 0.00012, "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25709,10 +25705,11 @@ "tpm": 10000000 }, "gemini/gemini-embedding-2": { - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, + "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_token_batches": 1e-07, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 2289de9a951..4a02bb1a638 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -74,6 +74,39 @@ def test_missing_cache_read_policy_preserves_billing(prompt_tokens, read_rate, s assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4]) +def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: + model_info: ModelInfo = { + "key": "gemini-embedding-2", + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": 2e-7, + "input_cost_per_audio_token": 6.5e-6, + "input_cost_per_audio_per_second": 0.00016, + "output_cost_per_token": 0.0, + "litellm_provider": "vertex_ai", + "mode": "embedding", + } + usage = Usage( + prompt_tokens=64, + completion_tokens=0, + total_tokens=64, + prompt_tokens_details=PromptTokensDetailsWrapper( + audio_tokens=64, + audio_length_seconds=2, + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gemini-embedding-2", + usage=usage, + custom_llm_provider="vertex_ai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(2 * 0.00016) + + def test_missing_cache_read_uses_off_peak_input_rate(): from datetime import datetime, timezone diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 86b3f0976ab..fbf86105e71 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -22,7 +22,6 @@ from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation from litellm.types.llms.vertex_ai import VertexAIBatchEmbeddingsResponseObject from litellm.types.utils import EmbeddingResponse - IMAGE_DATA_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII" GCS_URL = "gs://my-bucket/image.png" @@ -324,7 +323,7 @@ class TestProcessEmbedContentResponseUsage: ) assert result.usage.prompt_tokens == 258 assert result.usage.total_tokens == 258 - assert result.usage.prompt_tokens_details.image_count == 1 + assert result.usage.prompt_tokens_details.image_tokens == 258 prompt_cost, _ = generic_cost_per_token( model=self.MODEL, @@ -358,7 +357,7 @@ class TestProcessEmbedContentResponseUsage: ) assert prompt_cost > 0 - def test_video_modality_derives_seconds_and_text_floor(self): + def test_video_modality_preserves_token_count(self): response_json = { "embedding": {"values": [0.1]}, "usageMetadata": { @@ -374,10 +373,8 @@ class TestProcessEmbedContentResponseUsage: response_json=response_json, ) assert result.usage.prompt_tokens == 516 - assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx( - 2.0 - ) - assert result.usage.prompt_tokens_details.text_tokens == 1 + assert result.usage.prompt_tokens_details.video_tokens == 516 + assert result.usage.prompt_tokens_details.text_tokens == 0 def test_missing_usage_metadata_does_not_estimate_from_base64(self): response_json = {"embedding": {"values": [0.1, 0.2]}} @@ -400,8 +397,7 @@ class TestProcessEmbedContentResponseUsage: ) assert result.usage.prompt_tokens > 0 - def test_file_reference_image_billed_per_image_not_text(self): - """files/... image refs must bill per-image, not at the text token rate.""" + def test_file_reference_image_billed_per_image_token_rate(self): response_json = { "embedding": {"values": [0.1, 0.2, 0.3]}, "usageMetadata": { @@ -422,7 +418,7 @@ class TestProcessEmbedContentResponseUsage: } }, ) - assert result.usage.prompt_tokens_details.image_count == 1 + assert result.usage.prompt_tokens_details.image_tokens == 258 assert result.usage.prompt_tokens_details.text_tokens == 0 prompt_cost, _ = generic_cost_per_token( @@ -430,10 +426,10 @@ class TestProcessEmbedContentResponseUsage: usage=result.usage, custom_llm_provider="vertex_ai", ) - assert prompt_cost == pytest.approx(0.00012) + assert prompt_cost == pytest.approx(258 * 4.5e-7) def test_file_reference_non_image_not_counted_as_image(self): - """A files/... ref resolving to a non-image mime must not be image-counted.""" + """A files/... ref resolving to a non-image mime keeps audio token billing.""" response_json = { "embedding": {"values": [0.1, 0.2]}, "usageMetadata": { @@ -454,21 +450,18 @@ class TestProcessEmbedContentResponseUsage: } }, ) - assert result.usage.prompt_tokens_details.image_count == 0 assert result.usage.prompt_tokens_details.audio_tokens == 64 - assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx( - 2.0 - ) + assert result.usage.prompt_tokens_details.image_tokens == 0 prompt_cost, _ = generic_cost_per_token( model=self.MODEL, usage=result.usage, custom_llm_provider="vertex_ai", ) - assert prompt_cost == pytest.approx(2.0 * 0.00016) + assert prompt_cost == pytest.approx(64 * 6.5e-6) def test_video_plus_audio_does_not_double_bill_text(self): - """Video+audio responses must not get video tokens reassigned to text.""" + """Video and audio responses are billed from their respective token counts.""" response_json = { "embedding": {"values": [0.1]}, "usageMetadata": { @@ -486,18 +479,13 @@ class TestProcessEmbedContentResponseUsage: model=self.MODEL, response_json=response_json, ) - assert result.usage.prompt_tokens_details.text_tokens == 1 - assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx( - 2.0 - ) - assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx( - 2.0 - ) + assert result.usage.prompt_tokens_details.text_tokens == 0 + assert result.usage.prompt_tokens_details.video_tokens == 516 + assert result.usage.prompt_tokens_details.audio_tokens == 64 prompt_cost, _ = generic_cost_per_token( model=self.MODEL, usage=result.usage, custom_llm_provider="vertex_ai", ) - # 1 floor text token at 2e-7 + 2s of video at 7.9e-4 + 2s of audio at 1.6e-4 - assert prompt_cost == pytest.approx(1 * 2e-7 + 2 * 0.00079 + 2 * 0.00016) + assert prompt_cost == pytest.approx(516 * 1.2e-5 + 64 * 6.5e-6) From e26a4970dd0ba5efe277a5f42b51854daaf5da6d Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 00:42:21 +0000 Subject: [PATCH 02/22] fix(test): complete synthetic model metadata Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 4a02bb1a638..97d04a03a78 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -86,6 +86,7 @@ def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: "output_cost_per_token": 0.0, "litellm_provider": "vertex_ai", "mode": "embedding", + "supported_openai_params": None, } usage = Usage( prompt_tokens=64, From 0c91d9157c43ba7728b58393b6088641aa824367 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 00:45:08 +0000 Subject: [PATCH 03/22] refactor(vertex): drop unused resolved_files from embed response parsing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../gemini_embeddings/batch_embed_content_handler.py | 2 -- .../batch_embed_content_transformation.py | 3 +-- .../test_batch_embed_content_transformation.py | 12 ------------ 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py index f81d4ca777e..e09622ba236 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py @@ -268,7 +268,6 @@ class GoogleBatchEmbeddings(VertexLLM): model_response=model_response, model=model, response_json=_json_response, - resolved_files=resolved_files, ) else: _predictions: Final = VertexAIBatchEmbeddingsResponseObject(**_json_response) @@ -372,7 +371,6 @@ class GoogleBatchEmbeddings(VertexLLM): model_response=model_response, model=model, response_json=_json_response, - resolved_files=resolved_files, ) else: _predictions: Final = VertexAIBatchEmbeddingsResponseObject(**_json_response) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 8e120ab9fe6..f2cce775f3f 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -4,7 +4,7 @@ Transformation logic from OpenAI /v1/embeddings format to Google AI Studio /batc Why separate file? Make it easy to see how transformation works """ -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from typing import Final from pydantic import TypeAdapter, ValidationError @@ -356,7 +356,6 @@ def process_embed_content_response( model_response: EmbeddingResponse, model: str, response_json: dict, - resolved_files: Mapping[str, Mapping[str, str]] | None = None, ) -> EmbeddingResponse: """ Process Gemini embedContent response (single embedding for multimodal input). diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index fbf86105e71..0251a799b66 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -411,12 +411,6 @@ class TestProcessEmbedContentResponseUsage: model_response=EmbeddingResponse(), model=self.MODEL, response_json=response_json, - resolved_files={ - "files/img123": { - "mime_type": "image/png", - "uri": "https://example.com/img123", - } - }, ) assert result.usage.prompt_tokens_details.image_tokens == 258 assert result.usage.prompt_tokens_details.text_tokens == 0 @@ -443,12 +437,6 @@ class TestProcessEmbedContentResponseUsage: model_response=EmbeddingResponse(), model=self.MODEL, response_json=response_json, - resolved_files={ - "files/clip1": { - "mime_type": "audio/mpeg", - "uri": "https://example.com/clip1", - } - }, ) assert result.usage.prompt_tokens_details.audio_tokens == 64 assert result.usage.prompt_tokens_details.image_tokens == 0 From 6c9fe65608997dbfa85c35d01ad196f8b97c9a9d Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 00:47:30 +0000 Subject: [PATCH 04/22] style(cost): apply ruff formatting to modality guards Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/llm_cost_calc/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index a004f46b291..88ea4b602cc 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -957,8 +957,7 @@ def _calculate_input_cost( ### AUDIO COST if prompt_tokens_details["audio_tokens"] and not ( - prompt_tokens_details["audio_length_seconds"] - and model_info.get("input_cost_per_audio_per_second") is not None + prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None ): audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier) prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"]) @@ -976,8 +975,7 @@ def _calculate_input_cost( ### VIDEO TOKEN COST if prompt_tokens_details["video_tokens"] and not ( - prompt_tokens_details["video_length_seconds"] - and model_info.get("input_cost_per_video_per_second") is not None + prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None ): video_token_cost_key = "input_cost_per_video_token" if model_info.get(video_token_cost_key) is None: From e5845c17ffde232ee1e460b648fb223ea4561348 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 00:51:49 +0000 Subject: [PATCH 05/22] fix(vertex): bill image inputs at the image rate when usage lacks modality details Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../batch_embed_content_handler.py | 2 + .../batch_embed_content_transformation.py | 52 +++++++++++++++- ...test_batch_embed_content_transformation.py | 60 +++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py index e09622ba236..f81d4ca777e 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py @@ -268,6 +268,7 @@ class GoogleBatchEmbeddings(VertexLLM): model_response=model_response, model=model, response_json=_json_response, + resolved_files=resolved_files, ) else: _predictions: Final = VertexAIBatchEmbeddingsResponseObject(**_json_response) @@ -371,6 +372,7 @@ class GoogleBatchEmbeddings(VertexLLM): model_response=model_response, model=model, response_json=_json_response, + resolved_files=resolved_files, ) else: _predictions: Final = VertexAIBatchEmbeddingsResponseObject(**_json_response) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index f2cce775f3f..b61fb47cf5c 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -4,7 +4,7 @@ Transformation logic from OpenAI /v1/embeddings format to Google AI Studio /batc Why separate file? Make it easy to see how transformation works """ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Final from pydantic import TypeAdapter, ValidationError @@ -297,6 +297,7 @@ def transform_openai_input_gemini_embed_content( return request_body +_IMAGE_MIME_TYPES: Final = frozenset({"image/png", "image/jpeg"}) _usage_metadata_adapter: Final = TypeAdapter(UsageMetadata) @@ -309,6 +310,40 @@ def _parse_usage_metadata(raw_usage_metadata: object) -> UsageMetadata | None: return None +def _flatten_input(input: GeminiEmbeddingInput) -> tuple[str, ...]: + if isinstance(input, str): + return (input,) + return tuple(sub for element in input for sub in (element if isinstance(element, list) else [element])) + + +def _is_image_element( + element: str, + resolved_files: Mapping[str, Mapping[str, str]], +) -> bool: + if element.startswith("data:") and ";base64," in element: + try: + mime_type, _ = _parse_data_url(element) + except ValueError: + return False + return mime_type in _IMAGE_MIME_TYPES + if _is_gcs_url(element): + try: + return _infer_mime_type_from_gcs_url(element) in _IMAGE_MIME_TYPES + except ValueError: + return False + if _is_file_reference(element): + file_info: Final = resolved_files.get(element) + return file_info is not None and file_info.get("mime_type") in _IMAGE_MIME_TYPES + return False + + +def _count_input_images( + input: GeminiEmbeddingInput, + resolved_files: Mapping[str, Mapping[str, str]], +) -> int: + return sum(1 for element in _flatten_input(input) if _is_image_element(element, resolved_files)) + + def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int: return sum(detail["tokenCount"] for detail in details if detail["modality"] == modality) @@ -325,6 +360,7 @@ def _usage_from_embed_content_response( input: GeminiEmbeddingInput, model: str, raw_usage_metadata: object, + resolved_files: Mapping[str, Mapping[str, str]], ) -> Usage: usage_metadata: Final = _parse_usage_metadata(raw_usage_metadata) if usage_metadata is None: @@ -334,6 +370,17 @@ def _usage_from_embed_content_response( total_tokens: Final = usage_metadata.get("totalTokenCount") or prompt_tokens details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or () + if not details: + image_tokens: Final = prompt_tokens if _count_input_images(input, resolved_files) else 0 + return Usage( + prompt_tokens=prompt_tokens, + total_tokens=total_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=0, + image_tokens=image_tokens, + ), + ) + text_tokens: Final = _tokens_for_modality(details, "TEXT") audio_tokens: Final = _tokens_for_modality(details, "AUDIO") image_tokens: Final = _tokens_for_modality(details, "IMAGE") @@ -356,6 +403,7 @@ def process_embed_content_response( model_response: EmbeddingResponse, model: str, response_json: dict, + resolved_files: Mapping[str, Mapping[str, str]] | None = None, ) -> EmbeddingResponse: """ Process Gemini embedContent response (single embedding for multimodal input). @@ -365,6 +413,7 @@ def process_embed_content_response( model_response: EmbeddingResponse to populate model: Model name response_json: Raw JSON response from embedContent endpoint + resolved_files: Mapping of file references to resolved metadata Returns: EmbeddingResponse with single embedding @@ -386,6 +435,7 @@ def process_embed_content_response( input=input, model=model, raw_usage_metadata=response_json.get("usageMetadata"), + resolved_files=resolved_files or {}, ) return model_response diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 0251a799b66..df5903b9285 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -411,6 +411,12 @@ class TestProcessEmbedContentResponseUsage: model_response=EmbeddingResponse(), model=self.MODEL, response_json=response_json, + resolved_files={ + "files/img123": { + "mime_type": "image/png", + "uri": "https://example.com/img123", + } + }, ) assert result.usage.prompt_tokens_details.image_tokens == 258 assert result.usage.prompt_tokens_details.text_tokens == 0 @@ -437,6 +443,12 @@ class TestProcessEmbedContentResponseUsage: model_response=EmbeddingResponse(), model=self.MODEL, response_json=response_json, + resolved_files={ + "files/clip1": { + "mime_type": "audio/mpeg", + "uri": "https://example.com/clip1", + } + }, ) assert result.usage.prompt_tokens_details.audio_tokens == 64 assert result.usage.prompt_tokens_details.image_tokens == 0 @@ -477,3 +489,51 @@ class TestProcessEmbedContentResponseUsage: custom_llm_provider="vertex_ai", ) assert prompt_cost == pytest.approx(516 * 1.2e-5 + 64 * 6.5e-6) + + def test_image_without_modality_details_uses_image_rate(self): + response_json = { + "embedding": {"values": [0.1]}, + "usageMetadata": { + "promptTokenCount": 258, + "totalTokenCount": 258, + }, + } + result = process_embed_content_response( + input=IMAGE_DATA_URI, + model_response=EmbeddingResponse(), + model=self.MODEL, + response_json=response_json, + ) + assert result.usage.prompt_tokens_details.image_tokens == 258 + assert result.usage.prompt_tokens_details.text_tokens == 0 + + prompt_cost, _ = generic_cost_per_token( + model=self.MODEL, + usage=result.usage, + custom_llm_provider="vertex_ai", + ) + assert prompt_cost == pytest.approx(258 * 4.5e-7) + + def test_text_without_modality_details_uses_text_rate(self): + response_json = { + "embedding": {"values": [0.1]}, + "usageMetadata": { + "promptTokenCount": 12, + "totalTokenCount": 12, + }, + } + result = process_embed_content_response( + input="a short caption", + model_response=EmbeddingResponse(), + model=self.MODEL, + response_json=response_json, + ) + assert result.usage.prompt_tokens_details.text_tokens == 0 + assert result.usage.prompt_tokens_details.image_tokens == 0 + + prompt_cost, _ = generic_cost_per_token( + model=self.MODEL, + usage=result.usage, + custom_llm_provider="vertex_ai", + ) + assert prompt_cost == pytest.approx(12 * 2e-7) From e31c64d2e038fc0895cf02e45e2c6a798bba3cf3 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 00:54:58 +0000 Subject: [PATCH 06/22] fix(schema): sync model price schema with cost map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- model_prices_and_context_window.schema.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index d1ac3e67b2b..eff3f192b3c 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -375,6 +375,10 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "input_cost_per_video_token": { + "type": "number", + "minimum": 0 + }, "input_dbu_cost_per_token": { "type": "number", "minimum": 0 From ac8e1a355cf69830ec989fd19afb42a2bef78efd Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 01:00:32 +0000 Subject: [PATCH 07/22] test(vertex): load local pricing in embedding billing tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_batch_embed_content_transformation.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index df5903b9285..49f5167fd6b 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -10,6 +10,7 @@ Covers: import pytest +import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( _build_part_for_input, @@ -26,6 +27,15 @@ IMAGE_DATA_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+ GCS_URL = "gs://my-bucket/image.png" +@pytest.fixture(autouse=True) +def _local_model_cost_map(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + class TestIsMultimodalInput: def test_text_only_string(self): assert _is_multimodal_input("hello world") is False From 6cbed7b4c0ae643a429b0ebc7cb85a99e52b5b9e Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 01:10:27 +0000 Subject: [PATCH 08/22] fix(vertex): drop Final image_tokens redeclaration flagged by basedpyright Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../gemini_embeddings/batch_embed_content_transformation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index b61fb47cf5c..b618f6e5165 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -371,13 +371,12 @@ def _usage_from_embed_content_response( details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or () if not details: - image_tokens: Final = prompt_tokens if _count_input_images(input, resolved_files) else 0 return Usage( prompt_tokens=prompt_tokens, total_tokens=total_tokens, prompt_tokens_details=PromptTokensDetailsWrapper( text_tokens=0, - image_tokens=image_tokens, + image_tokens=prompt_tokens if _count_input_images(input, resolved_files) else 0, ), ) From 6a18105275223a39170e24d8fec96d123af82b32 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 01:16:00 +0000 Subject: [PATCH 09/22] fix(vertex): only bill image rate without modality details when every input is an image Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../batch_embed_content_transformation.py | 9 ++++---- ...test_batch_embed_content_transformation.py | 23 +++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index b618f6e5165..d669acecfd9 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -337,11 +337,12 @@ def _is_image_element( return False -def _count_input_images( +def _is_image_only_input( input: GeminiEmbeddingInput, resolved_files: Mapping[str, Mapping[str, str]], -) -> int: - return sum(1 for element in _flatten_input(input) if _is_image_element(element, resolved_files)) +) -> bool: + elements: Final = _flatten_input(input) + return bool(elements) and all(_is_image_element(element, resolved_files) for element in elements) def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int: @@ -376,7 +377,7 @@ def _usage_from_embed_content_response( total_tokens=total_tokens, prompt_tokens_details=PromptTokensDetailsWrapper( text_tokens=0, - image_tokens=prompt_tokens if _count_input_images(input, resolved_files) else 0, + image_tokens=prompt_tokens if _is_image_only_input(input, resolved_files) else 0, ), ) diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 49f5167fd6b..5dfeac6f469 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -524,6 +524,29 @@ class TestProcessEmbedContentResponseUsage: ) assert prompt_cost == pytest.approx(258 * 4.5e-7) + def test_mixed_text_and_image_without_modality_details_not_billed_as_image(self): + response_json = { + "embedding": {"values": [0.1]}, + "usageMetadata": { + "promptTokenCount": 270, + "totalTokenCount": 270, + }, + } + result = process_embed_content_response( + input=["a short caption", IMAGE_DATA_URI], + model_response=EmbeddingResponse(), + model=self.MODEL, + response_json=response_json, + ) + assert result.usage.prompt_tokens_details.image_tokens == 0 + + prompt_cost, _ = generic_cost_per_token( + model=self.MODEL, + usage=result.usage, + custom_llm_provider="vertex_ai", + ) + assert prompt_cost == pytest.approx(270 * 2e-7) + def test_text_without_modality_details_uses_text_rate(self): response_json = { "embedding": {"values": [0.1]}, From 4a8ec7b9d8c60896b28448b3bd87380617692d74 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 01:18:50 +0000 Subject: [PATCH 10/22] fix(cost): bill gemini-embedding-2-preview per token like the GA entries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 18 +++++++-------- model_prices_and_context_window.json | 18 +++++++-------- ...test_batch_embed_content_transformation.py | 22 +++++++++++++++++++ 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 74ca23fcc8a..9299a1493f8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25601,10 +25601,10 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" }, "gemini-embedding-2-preview": { - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, + "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25631,10 +25631,10 @@ "uses_embed_content": true }, "vertex_ai/gemini-embedding-2-preview": { - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, + "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25689,10 +25689,10 @@ }, "gemini/gemini-embedding-2-preview": { "deprecation_date": "2026-08-10", - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, + "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 74ca23fcc8a..9299a1493f8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25601,10 +25601,10 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models" }, "gemini-embedding-2-preview": { - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, + "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25631,10 +25631,10 @@ "uses_embed_content": true }, "vertex_ai/gemini-embedding-2-preview": { - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, + "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25689,10 +25689,10 @@ }, "gemini/gemini-embedding-2-preview": { "deprecation_date": "2026-08-10", - "input_cost_per_audio_per_second": 0.00016, - "input_cost_per_image": 0.00012, + "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, - "input_cost_per_video_per_second": 0.00079, + "input_cost_per_video_token": 1.2e-05, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 5dfeac6f469..926570d7929 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -500,6 +500,28 @@ class TestProcessEmbedContentResponseUsage: ) assert prompt_cost == pytest.approx(516 * 1.2e-5 + 64 * 6.5e-6) + def test_preview_alias_bills_audio_per_token(self): + response_json = { + "embedding": {"values": [0.1]}, + "usageMetadata": { + "promptTokenCount": 64, + "totalTokenCount": 64, + "promptTokensDetails": [{"modality": "AUDIO", "tokenCount": 64}], + }, + } + result = process_embed_content_response( + input="audio", + model_response=EmbeddingResponse(), + model="gemini-embedding-2-preview", + response_json=response_json, + ) + prompt_cost, _ = generic_cost_per_token( + model="gemini-embedding-2-preview", + usage=result.usage, + custom_llm_provider="vertex_ai", + ) + assert prompt_cost == pytest.approx(64 * 6.5e-6) + def test_image_without_modality_details_uses_image_rate(self): response_json = { "embedding": {"values": [0.1]}, From 27a486e4d320b4481c8aad6062f0c518d1c52ea4 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 01:34:17 +0000 Subject: [PATCH 11/22] test(cost): cover modality guards and image detection fallbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/test_llm_cost_calc_utils.py | 68 +++++++++++++++++++ ...test_batch_embed_content_transformation.py | 39 +++++++++++ 2 files changed, 107 insertions(+) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 97d04a03a78..854bc9bbb81 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -108,6 +108,74 @@ def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: assert prompt_cost == pytest.approx(2 * 0.00016) +def test_generic_cost_per_token_prefers_image_per_image_rate() -> None: + model_info: ModelInfo = { + "key": "gemini-embedding-2", + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": 2e-7, + "input_cost_per_image_token": 4.5e-7, + "input_cost_per_image": 0.00012, + "output_cost_per_token": 0.0, + "litellm_provider": "vertex_ai", + "mode": "embedding", + "supported_openai_params": None, + } + usage = Usage( + prompt_tokens=258, + completion_tokens=0, + total_tokens=258, + prompt_tokens_details=PromptTokensDetailsWrapper( + image_tokens=258, + image_count=1, + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gemini-embedding-2", + usage=usage, + custom_llm_provider="vertex_ai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(0.00012) + + +def test_generic_cost_per_token_prefers_video_per_second_rate() -> None: + model_info: ModelInfo = { + "key": "gemini-embedding-2", + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": 2e-7, + "input_cost_per_video_token": 1.2e-5, + "input_cost_per_video_per_second": 0.00079, + "output_cost_per_token": 0.0, + "litellm_provider": "vertex_ai", + "mode": "embedding", + "supported_openai_params": None, + } + usage = Usage( + prompt_tokens=516, + completion_tokens=0, + total_tokens=516, + prompt_tokens_details=PromptTokensDetailsWrapper( + video_tokens=516, + video_length_seconds=2, + ), + ) + + prompt_cost, _ = generic_cost_per_token( + model="gemini-embedding-2", + usage=usage, + custom_llm_provider="vertex_ai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(2 * 0.00079) + + def test_missing_cache_read_uses_off_peak_input_rate(): from datetime import datetime, timezone diff --git a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index 926570d7929..fd8c2a9cf6a 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -546,6 +546,45 @@ class TestProcessEmbedContentResponseUsage: ) assert prompt_cost == pytest.approx(258 * 4.5e-7) + @pytest.mark.parametrize( + "input_value,resolved_files,expected_image_tokens", + [ + (GCS_URL, {}, 258), + ("gs://my-bucket/clip.mp4", {}, 0), + ("gs://my-bucket/unknown.bin", {}, 0), + ("files/image-123", {"files/image-123": {"mime_type": "image/jpeg"}}, 258), + ("files/missing", {}, 0), + ("data:application/octet-stream;base64,abc", {}, 0), + ([[IMAGE_DATA_URI]], {}, 258), + ([], {}, 0), + ], + ) + def test_missing_modality_details_classifies_image_inputs(self, input_value, resolved_files, expected_image_tokens): + response_json = { + "embedding": {"values": [0.1]}, + "usageMetadata": { + "promptTokenCount": 258, + "totalTokenCount": 258, + }, + } + result = process_embed_content_response( + input=input_value, + model_response=EmbeddingResponse(), + model=self.MODEL, + response_json=response_json, + resolved_files=resolved_files, + ) + assert result.usage.prompt_tokens_details.image_tokens == expected_image_tokens + assert result.usage.prompt_tokens_details.text_tokens == 0 + + prompt_cost, _ = generic_cost_per_token( + model=self.MODEL, + usage=result.usage, + custom_llm_provider="vertex_ai", + ) + expected_rate = 4.5e-7 if expected_image_tokens else 2e-7 + assert prompt_cost == pytest.approx(258 * expected_rate) + def test_mixed_text_and_image_without_modality_details_not_billed_as_image(self): response_json = { "embedding": {"values": [0.1]}, From a28ea22ec131d1ce9f47af4dceb1faf7df7aa2f2 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 01:34:22 +0000 Subject: [PATCH 12/22] fix(cost): move gemini-embedding-2-preview to per-token rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 3 +++ model_prices_and_context_window.json | 3 +++ tests/test_litellm/test_utils.py | 11 +++++++---- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9299a1493f8..29243832c45 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25604,6 +25604,7 @@ "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, + "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, @@ -25634,6 +25635,7 @@ "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, + "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, @@ -25692,6 +25694,7 @@ "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, + "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, "litellm_provider": "gemini", "max_input_tokens": 8192, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9299a1493f8..29243832c45 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25604,6 +25604,7 @@ "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, + "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, @@ -25634,6 +25635,7 @@ "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, + "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, @@ -25692,6 +25694,7 @@ "input_cost_per_audio_token": 6.5e-06, "input_cost_per_image_token": 4.5e-07, "input_cost_per_token": 2e-07, + "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, "litellm_provider": "gemini", "max_input_tokens": 8192, diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 04d2d35e05f..1da53fba923 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2946,7 +2946,7 @@ def test_model_info_for_openrouter_kimi_k2_5(): def test_gemini_embedding_2_ga_in_cost_map(): - """GA and Vertex preview gemini-embedding-2 entries align with multimodal unit pricing.""" + """GA and Vertex preview gemini-embedding-2 entries align with multimodal token pricing.""" import json from pathlib import Path @@ -2968,9 +2968,12 @@ def test_gemini_embedding_2_ga_in_cost_map(): assert info.get("mode") == "embedding" assert info.get("supports_multimodal") is True assert info.get("input_cost_per_token") == 2e-07 - assert info.get("input_cost_per_image") == 0.00012 - assert info.get("input_cost_per_audio_per_second") == 0.00016 - assert info.get("input_cost_per_video_per_second") == 0.00079 + assert info.get("input_cost_per_audio_token") == 6.5e-06 + assert info.get("input_cost_per_image_token") == 4.5e-07 + assert info.get("input_cost_per_video_token") == 1.2e-05 + assert "input_cost_per_image" not in info + assert "input_cost_per_audio_per_second" not in info + assert "input_cost_per_video_per_second" not in info if provider in ("vertex_ai-embedding-models", "vertex_ai"): assert ( info.get("uses_embed_content") is True From ce83fac3515c36c927ed133fe48abd7dc1a3ee74 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 01:50:21 +0000 Subject: [PATCH 13/22] fix(cost): bill batch embeddings per modality token rate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ci_cd/generate_model_prices_schema.py | 3 ++ litellm/cost_calculator.py | 30 ++++++++++- ...odel_prices_and_context_window_backup.json | 18 +++++++ litellm/types/utils.py | 6 +++ litellm/utils.py | 3 ++ model_prices_and_context_window.json | 18 +++++++ model_prices_and_context_window.schema.json | 15 ++++++ tests/test_litellm/test_cost_calculator.py | 51 +++++++++++++++++++ tests/test_litellm/test_utils.py | 3 ++ 9 files changed, 146 insertions(+), 1 deletion(-) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index ab29b70bdd4..ee0e7f22eb1 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -172,7 +172,10 @@ COST_DESCRIPTIONS: dict[str, str] = { ), "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.", "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.", + "input_cost_per_audio_token_batches": "USD per audio prompt token via the provider's batch API.", + "input_cost_per_image_token_batches": "USD per image prompt token via the provider's batch API.", "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.", + "input_cost_per_video_token_batches": "USD per video prompt token via the provider's batch API.", "output_cost_per_token_batches": "USD per generated token via the provider's batch API.", } diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index f5319776213..cbb9a45ada9 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2337,7 +2337,35 @@ def batch_cost_calculator( total_prompt_cost = 0.0 total_completion_cost = 0.0 if input_cost_per_token_batches is not None: - total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches + batch_details: Final = parse_prompt_tokens_details(usage) + audio_tokens, image_tokens, video_tokens = ( + batch_details["audio_tokens"], + batch_details["image_tokens"], + batch_details["video_tokens"], + ) + modality_rates: Final = ( + cast(float, model_info.get("input_cost_per_audio_token_batches")) + if model_info.get("input_cost_per_audio_token_batches") is not None + else input_cost_per_token_batches, + cast(float, model_info.get("input_cost_per_image_token_batches")) + if model_info.get("input_cost_per_image_token_batches") is not None + else input_cost_per_token_batches, + cast(float, model_info.get("input_cost_per_video_token_batches")) + if model_info.get("input_cost_per_video_token_batches") is not None + else input_cost_per_token_batches, + ) + total_prompt_cost = sum( + tokens * rate + for tokens, rate in zip( + ( + max(cast(int, usage.prompt_tokens) - audio_tokens - image_tokens - video_tokens, 0), + audio_tokens, + image_tokens, + video_tokens, + ), + (input_cost_per_token_batches, *modality_rates), + ) + ) elif input_cost_per_token: details: Final = parse_prompt_tokens_details(usage) cache_read_tokens: Final = details["cache_hit_tokens"] diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 29243832c45..9f91cf82f41 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25602,10 +25602,13 @@ }, "gemini-embedding-2-preview": { "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25617,10 +25620,13 @@ }, "gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25633,10 +25639,13 @@ }, "vertex_ai/gemini-embedding-2-preview": { "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25649,10 +25658,13 @@ }, "vertex_ai/gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25692,10 +25704,13 @@ "gemini/gemini-embedding-2-preview": { "deprecation_date": "2026-08-10", "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25709,10 +25724,13 @@ }, "gemini/gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index fbfcc678de9..2e8b20edf7a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -283,8 +283,11 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_video_token: float | None # for gemini omni models with video input input_cost_per_audio_per_second: float | None # only for vertex ai models input_cost_per_video_per_second: float | None # only for vertex ai models + input_cost_per_audio_token_batches: ReadOnly[float | None] + input_cost_per_image_token_batches: ReadOnly[float | None] input_cost_per_second: float | None # for OpenAI Speech models input_cost_per_token_batches: float | None + input_cost_per_video_token_batches: ReadOnly[float | None] output_cost_per_token_batches: float | None output_cost_per_token: Required[float | None] output_cost_per_token_flex: float | None # OpenAI flex service tier pricing @@ -3583,7 +3586,10 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): input_cost_per_video_per_second_above_128k_tokens: float | None = None input_cost_per_video_per_second_above_15s_interval: float | None = None input_cost_per_video_per_second_above_8s_interval: float | None = None + input_cost_per_audio_token_batches: float | None = None + input_cost_per_image_token_batches: float | None = None input_cost_per_token_batches: float | None = None + input_cost_per_video_token_batches: float | None = None output_cost_per_token_batches: float | None = None output_cost_per_token_flex: float | None = None output_cost_per_token_priority: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index d4e3d58ba9f..b2715b41739 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5923,10 +5923,13 @@ def _get_model_info_helper( input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None), input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None), input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None), + input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None), + input_cost_per_image_token_batches=_model_info.get("input_cost_per_image_token_batches", None), input_cost_per_image=_model_info.get("input_cost_per_image", None), input_cost_per_audio_per_second=_model_info.get("input_cost_per_audio_per_second", None), input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None), input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"), + input_cost_per_video_token_batches=_model_info.get("input_cost_per_video_token_batches", None), output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"), output_cost_per_token=_output_cost_per_token, output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 29243832c45..9f91cf82f41 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25602,10 +25602,13 @@ }, "gemini-embedding-2-preview": { "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25617,10 +25620,13 @@ }, "gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai-embedding-models", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25633,10 +25639,13 @@ }, "vertex_ai/gemini-embedding-2-preview": { "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25649,10 +25658,13 @@ }, "vertex_ai/gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "vertex_ai", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25692,10 +25704,13 @@ "gemini/gemini-embedding-2-preview": { "deprecation_date": "2026-08-10", "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, @@ -25709,10 +25724,13 @@ }, "gemini/gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, + "input_cost_per_audio_token_batches": 3.25e-06, "input_cost_per_image_token": 4.5e-07, + "input_cost_per_image_token_batches": 2.25e-07, "input_cost_per_token": 2e-07, "input_cost_per_token_batches": 1e-07, "input_cost_per_video_token": 1.2e-05, + "input_cost_per_video_token_batches": 6e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, "max_tokens": 8192, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index eff3f192b3c..b7e0a9fd414 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -249,6 +249,11 @@ "type": "number", "minimum": 0 }, + "input_cost_per_audio_token_batches": { + "type": "number", + "minimum": 0, + "description": "USD per audio prompt token via the provider's batch API." + }, "input_cost_per_audio_token_priority": { "type": "number", "minimum": 0, @@ -276,6 +281,11 @@ "type": "number", "minimum": 0 }, + "input_cost_per_image_token_batches": { + "type": "number", + "minimum": 0, + "description": "USD per image prompt token via the provider's batch API." + }, "input_cost_per_pixel": { "type": "number", "minimum": 0 @@ -379,6 +389,11 @@ "type": "number", "minimum": 0 }, + "input_cost_per_video_token_batches": { + "type": "number", + "minimum": 0, + "description": "USD per video prompt token via the provider's batch API." + }, "input_dbu_cost_per_token": { "type": "number", "minimum": 0 diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 68e9b6143a0..8c3436d3108 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3909,6 +3909,57 @@ def _batch_cache_usage() -> Usage: ) +def test_batch_cost_calculator_prices_multimodal_tokens_at_modality_rates(): + from litellm.cost_calculator import batch_cost_calculator + + model_info: ModelInfo = { + "input_cost_per_token_batches": 1e-7, + "input_cost_per_audio_token_batches": 3.25e-6, + "input_cost_per_image_token_batches": 2.25e-7, + "input_cost_per_video_token_batches": 6e-6, + } + usage = Usage( + prompt_tokens=100, + completion_tokens=0, + total_tokens=100, + prompt_tokens_details=PromptTokensDetailsWrapper( + audio_tokens=64, + image_tokens=10, + video_tokens=6, + ), + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, + model="gemini-embedding-2", + custom_llm_provider="vertex_ai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(20 * 1e-7 + 64 * 3.25e-6 + 10 * 2.25e-7 + 6 * 6e-6) + + +def test_batch_cost_calculator_falls_back_to_text_batch_rate_for_modalities(): + from litellm.cost_calculator import batch_cost_calculator + + model_info: ModelInfo = {"input_cost_per_token_batches": 1e-7} + usage = Usage( + prompt_tokens=100, + completion_tokens=0, + total_tokens=100, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=64), + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, + model="gemini-embedding-2", + custom_llm_provider="vertex_ai", + model_info=model_info, + ) + + assert prompt_cost == pytest.approx(100 * 1e-7) + + def test_batch_cost_calculator_prices_cache_creation_tokens_at_cache_write_rate(): """ LIT-4008 regression: anthropic batch usage is dominated by cache tokens. diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 1da53fba923..02ffaee0543 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2971,6 +2971,9 @@ def test_gemini_embedding_2_ga_in_cost_map(): assert info.get("input_cost_per_audio_token") == 6.5e-06 assert info.get("input_cost_per_image_token") == 4.5e-07 assert info.get("input_cost_per_video_token") == 1.2e-05 + assert info.get("input_cost_per_audio_token_batches") == 3.25e-06 + assert info.get("input_cost_per_image_token_batches") == 2.25e-07 + assert info.get("input_cost_per_video_token_batches") == 6e-06 assert "input_cost_per_image" not in info assert "input_cost_per_audio_per_second" not in info assert "input_cost_per_video_per_second" not in info From 2b32f586c087ca94c3427c31eb83513c2a03c599 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 01:51:41 +0000 Subject: [PATCH 14/22] refactor(cost): extract batch modality rate lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index cbb9a45ada9..dce6b7299a2 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2278,6 +2278,19 @@ def default_video_cost_calculator( return 0.0 +def _batch_rate( + model_info: ModelInfo, + key: Literal[ + "input_cost_per_audio_token_batches", + "input_cost_per_image_token_batches", + "input_cost_per_video_token_batches", + ], + fallback: float, +) -> float: + rate: Final = model_info.get(key) + return fallback if rate is None else cast(float, rate) + + def batch_cost_calculator( usage: Usage, model: str, @@ -2344,15 +2357,9 @@ def batch_cost_calculator( batch_details["video_tokens"], ) modality_rates: Final = ( - cast(float, model_info.get("input_cost_per_audio_token_batches")) - if model_info.get("input_cost_per_audio_token_batches") is not None - else input_cost_per_token_batches, - cast(float, model_info.get("input_cost_per_image_token_batches")) - if model_info.get("input_cost_per_image_token_batches") is not None - else input_cost_per_token_batches, - cast(float, model_info.get("input_cost_per_video_token_batches")) - if model_info.get("input_cost_per_video_token_batches") is not None - else input_cost_per_token_batches, + _batch_rate(model_info, "input_cost_per_audio_token_batches", input_cost_per_token_batches), + _batch_rate(model_info, "input_cost_per_image_token_batches", input_cost_per_token_batches), + _batch_rate(model_info, "input_cost_per_video_token_batches", input_cost_per_token_batches), ) total_prompt_cost = sum( tokens * rate From c0c5044c45bac0a696cd270c442b00d29cb2756e Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 01:57:50 +0000 Subject: [PATCH 15/22] fix(batches): keep modality token details in raw vertex batch usage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/batches/batch_utils.py | 34 +++++++++++++++++-- .../test_litellm/batches/test_batch_utils.py | 32 +++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 +++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 87f8fd3946e..397bc0a35a2 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -3,14 +3,14 @@ from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass from dataclasses import replace as dataclasses_replace from enum import Enum -from typing import Any, Final, Literal +from typing import Any, Final, Literal, cast import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.types.llms.openai import Batch -from litellm.types.utils import ModelInfo, Usage +from litellm.types.utils import ModelInfo, PromptTokensDetailsWrapper, Usage from litellm.utils import token_counter @@ -310,6 +310,35 @@ def _aggregate_batch_cost_usage_models( ) +def _vertex_prompt_tokens_details( + usage_metadata: Mapping[str, object], +) -> PromptTokensDetailsWrapper | None: + raw_details: Final = usage_metadata.get("promptTokensDetails") + if not isinstance(raw_details, list): + return None + + raw_list: Final = cast(list[object], raw_details) + if not all(isinstance(detail, Mapping) for detail in raw_list): + return None + + details: Final = tuple(cast(Mapping[str, object], detail) for detail in raw_list) + normalized: Final = tuple( + (modality.upper(), token_count) + for detail in details + if isinstance(modality := detail.get("modality"), str) + and isinstance(token_count := detail.get("tokenCount"), int) + ) + if len(normalized) != len(details): + return None + + return PromptTokensDetailsWrapper( + text_tokens=sum(token_count for modality, token_count in normalized if modality in ("TEXT", "DOCUMENT")), + audio_tokens=sum(token_count for modality, token_count in normalized if modality == "AUDIO"), + image_tokens=sum(token_count for modality, token_count in normalized if modality == "IMAGE"), + video_tokens=sum(token_count for modality, token_count in normalized if modality == "VIDEO"), + ) + + def calculate_vertex_ai_batch_cost_and_usage( vertex_ai_batch_responses: list[dict], model_name: str | None = None, @@ -356,6 +385,7 @@ def calculate_vertex_ai_batch_cost_and_usage( prompt_tokens=_prompt, completion_tokens=_completion, total_tokens=_total, + prompt_tokens_details=_vertex_prompt_tokens_details(cast(Mapping[str, object], usage_metadata)), ) try: diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index 768ea332677..8b04d7af70a 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -695,6 +695,38 @@ def test_vertex_cost_and_usage_aggregation(monkeypatch): assert result.failed_requests == 0 +def test_vertex_batch_usage_preserves_modality_token_details(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/gemini-embedding-2", + { + "input_cost_per_token_batches": 1e-7, + "input_cost_per_audio_token_batches": 3.25e-6, + "input_cost_per_image_token_batches": 2.25e-7, + "input_cost_per_video_token_batches": 6e-6, + }, + ) + responses = [ + { + "response": { + "usageMetadata": { + "promptTokenCount": 84, + "candidatesTokenCount": 0, + "totalTokenCount": 84, + "promptTokensDetails": [ + {"modality": "AUDIO", "tokenCount": 64}, + {"modality": "TEXT", "tokenCount": 20}, + ], + } + } + } + ] + + result = bu.calculate_vertex_ai_batch_cost_and_usage(responses, "gemini-embedding-2") + + assert result.prompt_cost == pytest.approx(64 * 3.25e-6 + 20 * 1e-7) + + def test_vertex_cost_skips_none_response_body(monkeypatch): import litellm.cost_calculator as cc diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8aa05cf8c7c..73245806bb2 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29857,6 +29857,8 @@ export interface components { input_cost_per_audio_per_second_above_128k_tokens?: number | null; /** Input Cost Per Audio Token */ input_cost_per_audio_token?: number | null; + /** Input Cost Per Audio Token Batches */ + input_cost_per_audio_token_batches?: number | null; /** Input Cost Per Character */ input_cost_per_character?: number | null; /** Input Cost Per Character Above 128K Tokens */ @@ -29867,6 +29869,8 @@ export interface components { input_cost_per_image_above_128k_tokens?: number | null; /** Input Cost Per Image Token */ input_cost_per_image_token?: number | null; + /** Input Cost Per Image Token Batches */ + input_cost_per_image_token_batches?: number | null; /** Input Cost Per Pixel */ input_cost_per_pixel?: number | null; /** Input Cost Per Query */ @@ -29909,6 +29913,8 @@ export interface components { input_cost_per_video_per_second_above_8s_interval?: number | null; /** Input Cost Per Video Token */ input_cost_per_video_token?: number | null; + /** Input Cost Per Video Token Batches */ + input_cost_per_video_token_batches?: number | null; /** Itpm */ itpm?: number | null; /** Keepalive Seconds */ @@ -40071,6 +40077,8 @@ export interface components { input_cost_per_audio_per_second_above_128k_tokens?: number | null; /** Input Cost Per Audio Token */ input_cost_per_audio_token?: number | null; + /** Input Cost Per Audio Token Batches */ + input_cost_per_audio_token_batches?: number | null; /** Input Cost Per Character */ input_cost_per_character?: number | null; /** Input Cost Per Character Above 128K Tokens */ @@ -40081,6 +40089,8 @@ export interface components { input_cost_per_image_above_128k_tokens?: number | null; /** Input Cost Per Image Token */ input_cost_per_image_token?: number | null; + /** Input Cost Per Image Token Batches */ + input_cost_per_image_token_batches?: number | null; /** Input Cost Per Pixel */ input_cost_per_pixel?: number | null; /** Input Cost Per Query */ @@ -40123,6 +40133,8 @@ export interface components { input_cost_per_video_per_second_above_8s_interval?: number | null; /** Input Cost Per Video Token */ input_cost_per_video_token?: number | null; + /** Input Cost Per Video Token Batches */ + input_cost_per_video_token_batches?: number | null; /** Itpm */ itpm?: number | null; /** Keepalive Seconds */ From ca7364fb0568a1d7f6b085529d45da2487fcb62c Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 02:00:48 +0000 Subject: [PATCH 16/22] fix(batches): avoid strict lint violation in usage parser Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/batches/batch_utils.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index 397bc0a35a2..acc0036f27d 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -3,7 +3,7 @@ from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass from dataclasses import replace as dataclasses_replace from enum import Enum -from typing import Any, Final, Literal, cast +from typing import Any, Final, Literal import litellm from litellm._logging import verbose_logger @@ -317,18 +317,18 @@ def _vertex_prompt_tokens_details( if not isinstance(raw_details, list): return None - raw_list: Final = cast(list[object], raw_details) - if not all(isinstance(detail, Mapping) for detail in raw_list): - return None + def _normalize(detail: object) -> tuple[str, int] | None: + if not isinstance(detail, Mapping): + return None + modality: Final = detail.get("modality") + token_count: Final = detail.get("tokenCount") + if not isinstance(modality, str) or not isinstance(token_count, int): + return None + return modality.upper(), token_count - details: Final = tuple(cast(Mapping[str, object], detail) for detail in raw_list) - normalized: Final = tuple( - (modality.upper(), token_count) - for detail in details - if isinstance(modality := detail.get("modality"), str) - and isinstance(token_count := detail.get("tokenCount"), int) - ) - if len(normalized) != len(details): + parsed_details: Final = tuple(_normalize(detail) for detail in raw_details) + normalized: Final = tuple(detail for detail in parsed_details if detail is not None) + if len(normalized) != len(parsed_details): return None return PromptTokensDetailsWrapper( @@ -385,7 +385,7 @@ def calculate_vertex_ai_batch_cost_and_usage( prompt_tokens=_prompt, completion_tokens=_completion, total_tokens=_total, - prompt_tokens_details=_vertex_prompt_tokens_details(cast(Mapping[str, object], usage_metadata)), + prompt_tokens_details=_vertex_prompt_tokens_details(usage_metadata), ) try: From a5fc880c907356a79843987eecc6aa170263271c Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 02:03:32 +0000 Subject: [PATCH 17/22] refactor(vertex): move batch usage modality parsing under llms Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/batches/batch_utils.py | 34 ++----------------- .../llms/vertex_ai/batches/transformation.py | 32 ++++++++++++++++- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index acc0036f27d..26b4318da2d 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -9,8 +9,9 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details +from litellm.llms.vertex_ai.batches.transformation import vertex_prompt_tokens_details from litellm.types.llms.openai import Batch -from litellm.types.utils import ModelInfo, PromptTokensDetailsWrapper, Usage +from litellm.types.utils import ModelInfo, Usage from litellm.utils import token_counter @@ -310,35 +311,6 @@ def _aggregate_batch_cost_usage_models( ) -def _vertex_prompt_tokens_details( - usage_metadata: Mapping[str, object], -) -> PromptTokensDetailsWrapper | None: - raw_details: Final = usage_metadata.get("promptTokensDetails") - if not isinstance(raw_details, list): - return None - - def _normalize(detail: object) -> tuple[str, int] | None: - if not isinstance(detail, Mapping): - return None - modality: Final = detail.get("modality") - token_count: Final = detail.get("tokenCount") - if not isinstance(modality, str) or not isinstance(token_count, int): - return None - return modality.upper(), token_count - - parsed_details: Final = tuple(_normalize(detail) for detail in raw_details) - normalized: Final = tuple(detail for detail in parsed_details if detail is not None) - if len(normalized) != len(parsed_details): - return None - - return PromptTokensDetailsWrapper( - text_tokens=sum(token_count for modality, token_count in normalized if modality in ("TEXT", "DOCUMENT")), - audio_tokens=sum(token_count for modality, token_count in normalized if modality == "AUDIO"), - image_tokens=sum(token_count for modality, token_count in normalized if modality == "IMAGE"), - video_tokens=sum(token_count for modality, token_count in normalized if modality == "VIDEO"), - ) - - def calculate_vertex_ai_batch_cost_and_usage( vertex_ai_batch_responses: list[dict], model_name: str | None = None, @@ -385,7 +357,7 @@ def calculate_vertex_ai_batch_cost_and_usage( prompt_tokens=_prompt, completion_tokens=_completion, total_tokens=_total, - prompt_tokens_details=_vertex_prompt_tokens_details(usage_metadata), + prompt_tokens_details=vertex_prompt_tokens_details(usage_metadata), ) try: diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index e63c80dd3cf..f5f1ab2068a 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Any, Final from urllib.parse import unquote @@ -8,7 +9,36 @@ from litellm.llms.vertex_ai.common_utils import ( ) from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest from litellm.types.llms.vertex_ai import * -from litellm.types.utils import LiteLLMBatch +from litellm.types.utils import LiteLLMBatch, PromptTokensDetailsWrapper + + +def vertex_prompt_tokens_details( + usage_metadata: Mapping[str, object], +) -> PromptTokensDetailsWrapper | None: + raw_details: Final = usage_metadata.get("promptTokensDetails") + if not isinstance(raw_details, list): + return None + + def _normalize(detail: object) -> tuple[str, int] | None: + if not isinstance(detail, Mapping): + return None + modality: Final = detail.get("modality") + token_count: Final = detail.get("tokenCount") + if not isinstance(modality, str) or not isinstance(token_count, int): + return None + return modality.upper(), token_count + + parsed_details: Final = tuple(_normalize(detail) for detail in raw_details) + normalized: Final = tuple(detail for detail in parsed_details if detail is not None) + if len(normalized) != len(parsed_details): + return None + + return PromptTokensDetailsWrapper( + text_tokens=sum(token_count for modality, token_count in normalized if modality in ("TEXT", "DOCUMENT")), + audio_tokens=sum(token_count for modality, token_count in normalized if modality == "AUDIO"), + image_tokens=sum(token_count for modality, token_count in normalized if modality == "IMAGE"), + video_tokens=sum(token_count for modality, token_count in normalized if modality == "VIDEO"), + ) class VertexAIBatchTransformation: From bf1bdb3045670322350e1789f210da8e41c5a8e9 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 02:07:44 +0000 Subject: [PATCH 18/22] fix(ci): keep cost map schema generated by the base branch generator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ci_cd/generate_model_prices_schema.py | 3 --- model_prices_and_context_window.schema.json | 9 +++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index ee0e7f22eb1..ab29b70bdd4 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -172,10 +172,7 @@ COST_DESCRIPTIONS: dict[str, str] = { ), "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.", "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.", - "input_cost_per_audio_token_batches": "USD per audio prompt token via the provider's batch API.", - "input_cost_per_image_token_batches": "USD per image prompt token via the provider's batch API.", "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.", - "input_cost_per_video_token_batches": "USD per video prompt token via the provider's batch API.", "output_cost_per_token_batches": "USD per generated token via the provider's batch API.", } diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index b7e0a9fd414..c2490041cf7 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -251,8 +251,7 @@ }, "input_cost_per_audio_token_batches": { "type": "number", - "minimum": 0, - "description": "USD per audio prompt token via the provider's batch API." + "minimum": 0 }, "input_cost_per_audio_token_priority": { "type": "number", @@ -283,8 +282,7 @@ }, "input_cost_per_image_token_batches": { "type": "number", - "minimum": 0, - "description": "USD per image prompt token via the provider's batch API." + "minimum": 0 }, "input_cost_per_pixel": { "type": "number", @@ -391,8 +389,7 @@ }, "input_cost_per_video_token_batches": { "type": "number", - "minimum": 0, - "description": "USD per video prompt token via the provider's batch API." + "minimum": 0 }, "input_dbu_cost_per_token": { "type": "number", From a5f00b9189fa1dad1515800b0e0797a2fa411099 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 02:15:22 +0000 Subject: [PATCH 19/22] fix(cost): drop unnecessary cast in batch rate lookup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index dce6b7299a2..39af725a231 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2288,7 +2288,7 @@ def _batch_rate( fallback: float, ) -> float: rate: Final = model_info.get(key) - return fallback if rate is None else cast(float, rate) + return fallback if rate is None else rate def batch_cost_calculator( From 168d0d3d997e11b39240479cf47ddd1675ba22c2 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 02:26:19 +0000 Subject: [PATCH 20/22] fix(cost): drop remaining unnecessary cast in batch cost calculator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 39af725a231..3dc6d81256b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2365,7 +2365,7 @@ def batch_cost_calculator( tokens * rate for tokens, rate in zip( ( - max(cast(int, usage.prompt_tokens) - audio_tokens - image_tokens - video_tokens, 0), + max((usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens, 0), audio_tokens, image_tokens, video_tokens, From 7761d044509e0ab0dae37ed6ded7835d4d06d7e9 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 02:37:51 +0000 Subject: [PATCH 21/22] test(vertex): cover malformed batch usage details Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../vertex_ai/batches/test_transformation.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py b/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py index 232c6413e78..e6126b02790 100644 --- a/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/batches/test_transformation.py @@ -17,9 +17,9 @@ from unittest.mock import patch import pytest - from litellm.llms.vertex_ai.batches.transformation import ( # noqa: E402 VertexAIBatchTransformation, + vertex_prompt_tokens_details, ) from litellm.llms.vertex_ai.common_utils import ( # noqa: E402 VertexAIError, @@ -41,6 +41,22 @@ ENDPOINT_INPUT_FILE = ( ) +def test_vertex_prompt_tokens_details_rejects_malformed_details(): + assert vertex_prompt_tokens_details({"promptTokensDetails": [1]}) is None + assert vertex_prompt_tokens_details({"promptTokensDetails": [{"modality": "AUDIO"}]}) is None + assert ( + vertex_prompt_tokens_details( + { + "promptTokensDetails": [ + {"modality": "AUDIO", "tokenCount": 1}, + "malformed", + ] + } + ) + is None + ) + + # =========================================================================== # # transform_openai_batch_request_to_vertex_ai_batch_request # =========================================================================== # From 954dfa6ba74882eecb3109438c993d769c636c92 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 02:38:53 +0000 Subject: [PATCH 22/22] test(utils): allow modality batch cost fields in cost map schema test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 02ffaee0543..d5feda6f892 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -892,7 +892,10 @@ def validate_model_cost_values(model_data, exceptions=None): "input_cost_per_video_per_second_above_8s_interval", "input_cost_per_video_per_second_above_15s_interval", "input_cost_per_video_per_second_above_128k_tokens", + "input_cost_per_audio_token_batches", + "input_cost_per_image_token_batches", "input_cost_per_token_batches", + "input_cost_per_video_token_batches", "output_cost_per_token_batches", "input_cost_per_token_cache_hit", "cache_creation_input_token_cost", @@ -1041,7 +1044,10 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_second": {"type": "number"}, "input_cost_per_token": {"type": "number"}, "input_cost_per_token_above_128k_tokens": {"type": "number"}, + "input_cost_per_audio_token_batches": {"type": "number"}, + "input_cost_per_image_token_batches": {"type": "number"}, "input_cost_per_token_batches": {"type": "number"}, + "input_cost_per_video_token_batches": {"type": "number"}, "input_cost_per_token_cache_hit": {"type": "number"}, "input_cost_per_video_per_second": {"type": "number"}, "input_cost_per_video_per_second_above_8s_interval": {"type": "number"},