scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-16 23:41:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

Merge pull request #41157 from BerriAI/litellm_gemini_embedding_2_per_token_billing

Browse source 
fix(cost): bill gemini-embedding-2 per token and stop double charging audio
This commit is contained in:
kerry-berri 2026-09-14 19:49:11 -07:00 • committed by GitHub
commit 03a4c806c4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 581 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/batches/batch_utils.py
View file

@ -9,6 +9,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.llms.vertex_ai.batches.transformation import vertex_prompt_tokens_details
from litellm.types.llms.openai import Batch
from litellm.types.utils import ModelInfo, Usage
from litellm.utils import token_counter
@ -356,6 +357,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
prompt_tokens=_prompt,
completion_tokens=_completion,
total_tokens=_total,
prompt_tokens_details=vertex_prompt_tokens_details(usage_metadata),
)
try:

37
litellm/cost_calculator.py
View file

@ -2278,6 +2278,19 @@ def default_video_cost_calculator(
return 0.0
def _batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
fallback: float,
) -> float:
rate: Final = model_info.get(key)
return fallback if rate is None else rate
def batch_cost_calculator(
usage: Usage,
model: str,
@ -2337,7 +2350,29 @@ def batch_cost_calculator(
total_prompt_cost = 0.0
total_completion_cost = 0.0
if input_cost_per_token_batches is not None:
total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
batch_details: Final = parse_prompt_tokens_details(usage)
audio_tokens, image_tokens, video_tokens = (
batch_details["audio_tokens"],
batch_details["image_tokens"],
batch_details["video_tokens"],
)
modality_rates: Final = (
_batch_rate(model_info, "input_cost_per_audio_token_batches", input_cost_per_token_batches),
_batch_rate(model_info, "input_cost_per_image_token_batches", input_cost_per_token_batches),
_batch_rate(model_info, "input_cost_per_video_token_batches", input_cost_per_token_batches),
)
total_prompt_cost = sum(
tokens * rate
for tokens, rate in zip(
(
max((usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens, 0),
audio_tokens,
image_tokens,
video_tokens,
),
(input_cost_per_token_batches, *modality_rates),
)
)
elif input_cost_per_token:
details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens: Final = details["cache_hit_tokens"]

12
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -956,12 +956,16 @@ def _calculate_input_cost(
)
### AUDIO COST
if prompt_tokens_details["audio_tokens"]:
if prompt_tokens_details["audio_tokens"] and not (
prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None
):
audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
### IMAGE TOKEN COST
if prompt_tokens_details["image_tokens"]:
if prompt_tokens_details["image_tokens"] and not (
prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None
):
# For image token costs:
# First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
image_token_cost_key = "input_cost_per_image_token"
@ -970,7 +974,9 @@ def _calculate_input_cost(
prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
### VIDEO TOKEN COST
if prompt_tokens_details["video_tokens"]:
if prompt_tokens_details["video_tokens"] and not (
prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None
):
video_token_cost_key = "input_cost_per_video_token"
if model_info.get(video_token_cost_key) is None:
video_token_cost_key = "input_cost_per_token"

32
litellm/llms/vertex_ai/batches/transformation.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Any, Final
from urllib.parse import unquote
@ -8,7 +9,36 @@ from litellm.llms.vertex_ai.common_utils import (
)
from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest
from litellm.types.llms.vertex_ai import *
from litellm.types.utils import LiteLLMBatch
from litellm.types.utils import LiteLLMBatch, PromptTokensDetailsWrapper
def vertex_prompt_tokens_details(
usage_metadata: Mapping[str, object],
) -> PromptTokensDetailsWrapper | None:
raw_details: Final = usage_metadata.get("promptTokensDetails")
if not isinstance(raw_details, list):
return None
def _normalize(detail: object) -> tuple[str, int] | None:
if not isinstance(detail, Mapping):
return None
modality: Final = detail.get("modality")
token_count: Final = detail.get("tokenCount")
if not isinstance(modality, str) or not isinstance(token_count, int):
return None
return modality.upper(), token_count
parsed_details: Final = tuple(_normalize(detail) for detail in raw_details)
normalized: Final = tuple(detail for detail in parsed_details if detail is not None)
if len(normalized) != len(parsed_details):
return None
return PromptTokensDetailsWrapper(
text_tokens=sum(token_count for modality, token_count in normalized if modality in ("TEXT", "DOCUMENT")),
audio_tokens=sum(token_count for modality, token_count in normalized if modality == "AUDIO"),
image_tokens=sum(token_count for modality, token_count in normalized if modality == "IMAGE"),
video_tokens=sum(token_count for modality, token_count in normalized if modality == "VIDEO"),
)
class VertexAIBatchTransformation:

41
litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py
View file

@ -298,8 +298,6 @@ def transform_openai_input_gemini_embed_content(
_IMAGE_MIME_TYPES: Final = frozenset({"image/png", "image/jpeg"})
_VIDEO_TOKENS_PER_SECOND: Final = 258.0
_AUDIO_TOKENS_PER_SECOND: Final = 32.0
_usage_metadata_adapter: Final = TypeAdapter(UsageMetadata)
@ -339,11 +337,12 @@ def _is_image_element(
return False
def _count_input_images(
def _is_image_only_input(
input: GeminiEmbeddingInput,
resolved_files: Mapping[str, Mapping[str, str]],
) -> int:
return sum(1 for element in _flatten_input(input) if _is_image_element(element, resolved_files))
) -> bool:
elements: Final = _flatten_input(input)
return bool(elements) and all(_is_image_element(element, resolved_files) for element in elements)
def _tokens_for_modality(details: Sequence[PromptTokensDetails], modality: str) -> int:
@ -372,30 +371,29 @@ def _usage_from_embed_content_response(
total_tokens: Final = usage_metadata.get("totalTokenCount") or prompt_tokens
details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or ()
if not details:
return Usage(
prompt_tokens=prompt_tokens,
total_tokens=total_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=0,
image_tokens=prompt_tokens if _is_image_only_input(input, resolved_files) else 0,
),
)
text_tokens: Final = _tokens_for_modality(details, "TEXT")
audio_tokens: Final = _tokens_for_modality(details, "AUDIO")
image_tokens: Final = _tokens_for_modality(details, "IMAGE")
video_tokens: Final = _tokens_for_modality(details, "VIDEO")
image_count: Final = _count_input_images(input, resolved_files)
video_length_seconds: Final = video_tokens / _VIDEO_TOKENS_PER_SECOND if video_tokens > 0 else 0.0
audio_length_seconds: Final = audio_tokens / _AUDIO_TOKENS_PER_SECOND if audio_tokens > 0 else 0.0
# generic_cost_per_token rewrites text_tokens to the full prompt minus
# other modalities when both text_tokens and image_count are zero. For
# video, that misallocates video tokens to text; a 1-token floor sidesteps
# the rewrite and keeps billing on input_cost_per_video_per_second.
needs_video_text_floor: Final = video_length_seconds > 0 and text_tokens == 0 and image_count == 0
resolved_text_tokens: Final = 1 if needs_video_text_floor else text_tokens
return Usage(
prompt_tokens=prompt_tokens,
total_tokens=total_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=resolved_text_tokens,
text_tokens=text_tokens,
audio_tokens=audio_tokens,
image_count=image_count,
video_length_seconds=video_length_seconds,
audio_length_seconds=audio_length_seconds,
image_tokens=image_tokens,
video_tokens=video_tokens,
),
)
@ -415,8 +413,7 @@ def process_embed_content_response(
model_response: EmbeddingResponse to populate
model: Model name
response_json: Raw JSON response from embedContent endpoint
resolved_files: Mapping of file references (files/abc) to {mime_type, uri},
used to bill resolved image references at the per-image rate
resolved_files: Mapping of file references to resolved metadata
Returns:
EmbeddingResponse with single embedding

54
litellm/model_prices_and_context_window_backup.json
View file

@ -25601,10 +25601,14 @@
"source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models"
},
"gemini-embedding-2-preview": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25615,13 +25619,14 @@
"uses_embed_content": true
},
"gemini-embedding-2": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25633,10 +25638,14 @@
"uses_embed_content": true
},
"vertex_ai/gemini-embedding-2-preview": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25648,13 +25657,14 @@
"uses_embed_content": true
},
"vertex_ai/gemini-embedding-2": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25693,10 +25703,14 @@
},
"gemini/gemini-embedding-2-preview": {
"deprecation_date": "2026-08-10",
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25709,10 +25723,14 @@
"tpm": 10000000
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,

6
litellm/types/utils.py
View file

@ -283,8 +283,11 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_video_token: float | None # for gemini omni models with video input
input_cost_per_audio_per_second: float | None # only for vertex ai models
input_cost_per_video_per_second: float | None # only for vertex ai models
input_cost_per_audio_token_batches: ReadOnly[float | None]
input_cost_per_image_token_batches: ReadOnly[float | None]
input_cost_per_second: float | None # for OpenAI Speech models
input_cost_per_token_batches: float | None
input_cost_per_video_token_batches: ReadOnly[float | None]
output_cost_per_token_batches: float | None
output_cost_per_token: Required[float | None]
output_cost_per_token_flex: float | None # OpenAI flex service tier pricing
@ -3583,7 +3586,10 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
input_cost_per_video_per_second_above_128k_tokens: float | None = None
input_cost_per_video_per_second_above_15s_interval: float | None = None
input_cost_per_video_per_second_above_8s_interval: float | None = None
input_cost_per_audio_token_batches: float | None = None
input_cost_per_image_token_batches: float | None = None
input_cost_per_token_batches: float | None = None
input_cost_per_video_token_batches: float | None = None
output_cost_per_token_batches: float | None = None
output_cost_per_token_flex: float | None = None
output_cost_per_token_priority: float | None = None

3
litellm/utils.py
View file

@ -5923,10 +5923,13 @@ def _get_model_info_helper(
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None),
input_cost_per_image_token_batches=_model_info.get("input_cost_per_image_token_batches", None),
input_cost_per_image=_model_info.get("input_cost_per_image", None),
input_cost_per_audio_per_second=_model_info.get("input_cost_per_audio_per_second", None),
input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None),
input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"),
input_cost_per_video_token_batches=_model_info.get("input_cost_per_video_token_batches", None),
output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"),
output_cost_per_token=_output_cost_per_token,
output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None),

54
model_prices_and_context_window.json
View file

@ -25601,10 +25601,14 @@
"source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models"
},
"gemini-embedding-2-preview": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25615,13 +25619,14 @@
"uses_embed_content": true
},
"gemini-embedding-2": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25633,10 +25638,14 @@
"uses_embed_content": true
},
"vertex_ai/gemini-embedding-2-preview": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25648,13 +25657,14 @@
"uses_embed_content": true
},
"vertex_ai/gemini-embedding-2": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25693,10 +25703,14 @@
},
"gemini/gemini-embedding-2-preview": {
"deprecation_date": "2026-08-10",
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25709,10 +25723,14 @@
"tpm": 10000000
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_image": 0.00012,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,

16
model_prices_and_context_window.schema.json
View file

@ -249,6 +249,10 @@
"type": "number",
"minimum": 0
},
"input_cost_per_audio_token_batches": {
"type": "number",
"minimum": 0
},
"input_cost_per_audio_token_priority": {
"type": "number",
"minimum": 0,
@ -276,6 +280,10 @@
"type": "number",
"minimum": 0
},
"input_cost_per_image_token_batches": {
"type": "number",
"minimum": 0
},
"input_cost_per_pixel": {
"type": "number",
"minimum": 0
@ -375,6 +383,14 @@
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"input_cost_per_video_token": {
"type": "number",
"minimum": 0
},
"input_cost_per_video_token_batches": {
"type": "number",
"minimum": 0
},
"input_dbu_cost_per_token": {
"type": "number",
"minimum": 0

32
tests/test_litellm/batches/test_batch_utils.py
View file

@ -695,6 +695,38 @@ def test_vertex_cost_and_usage_aggregation(monkeypatch):
assert result.failed_requests == 0
def test_vertex_batch_usage_preserves_modality_token_details(monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
"vertex_ai/gemini-embedding-2",
{
"input_cost_per_token_batches": 1e-7,
"input_cost_per_audio_token_batches": 3.25e-6,
"input_cost_per_image_token_batches": 2.25e-7,
"input_cost_per_video_token_batches": 6e-6,
},
)
responses = [
{
"response": {
"usageMetadata": {
"promptTokenCount": 84,
"candidatesTokenCount": 0,
"totalTokenCount": 84,
"promptTokensDetails": [
{"modality": "AUDIO", "tokenCount": 64},
{"modality": "TEXT", "tokenCount": 20},
],
}
}
}
]
result = bu.calculate_vertex_ai_batch_cost_and_usage(responses, "gemini-embedding-2")
assert result.prompt_cost == pytest.approx(64 * 3.25e-6 + 20 * 1e-7)
def test_vertex_cost_skips_none_response_body(monkeypatch):
import litellm.cost_calculator as cc

102
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -74,6 +74,108 @@ def test_missing_cache_read_policy_preserves_billing(prompt_tokens, read_rate, s
assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4])
def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None:
model_info: ModelInfo = {
"key": "gemini-embedding-2",
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
"input_cost_per_token": 2e-7,
"input_cost_per_audio_token": 6.5e-6,
"input_cost_per_audio_per_second": 0.00016,
"output_cost_per_token": 0.0,
"litellm_provider": "vertex_ai",
"mode": "embedding",
"supported_openai_params": None,
}
usage = Usage(
prompt_tokens=64,
completion_tokens=0,
total_tokens=64,
prompt_tokens_details=PromptTokensDetailsWrapper(
audio_tokens=64,
audio_length_seconds=2,
),
)
prompt_cost, _ = generic_cost_per_token(
model="gemini-embedding-2",
usage=usage,
custom_llm_provider="vertex_ai",
model_info=model_info,
)
assert prompt_cost == pytest.approx(2 * 0.00016)
def test_generic_cost_per_token_prefers_image_per_image_rate() -> None:
model_info: ModelInfo = {
"key": "gemini-embedding-2",
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
"input_cost_per_token": 2e-7,
"input_cost_per_image_token": 4.5e-7,
"input_cost_per_image": 0.00012,
"output_cost_per_token": 0.0,
"litellm_provider": "vertex_ai",
"mode": "embedding",
"supported_openai_params": None,
}
usage = Usage(
prompt_tokens=258,
completion_tokens=0,
total_tokens=258,
prompt_tokens_details=PromptTokensDetailsWrapper(
image_tokens=258,
image_count=1,
),
)
prompt_cost, _ = generic_cost_per_token(
model="gemini-embedding-2",
usage=usage,
custom_llm_provider="vertex_ai",
model_info=model_info,
)
assert prompt_cost == pytest.approx(0.00012)
def test_generic_cost_per_token_prefers_video_per_second_rate() -> None:
model_info: ModelInfo = {
"key": "gemini-embedding-2",
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
"input_cost_per_token": 2e-7,
"input_cost_per_video_token": 1.2e-5,
"input_cost_per_video_per_second": 0.00079,
"output_cost_per_token": 0.0,
"litellm_provider": "vertex_ai",
"mode": "embedding",
"supported_openai_params": None,
}
usage = Usage(
prompt_tokens=516,
completion_tokens=0,
total_tokens=516,
prompt_tokens_details=PromptTokensDetailsWrapper(
video_tokens=516,
video_length_seconds=2,
),
)
prompt_cost, _ = generic_cost_per_token(
model="gemini-embedding-2",
usage=usage,
custom_llm_provider="vertex_ai",
model_info=model_info,
)
assert prompt_cost == pytest.approx(2 * 0.00079)
def test_missing_cache_read_uses_off_peak_input_rate():
from datetime import datetime, timezone

18
tests/test_litellm/llms/vertex_ai/batches/test_transformation.py
View file

@ -17,9 +17,9 @@ from unittest.mock import patch
import pytest
from litellm.llms.vertex_ai.batches.transformation import ( # noqa: E402
VertexAIBatchTransformation,
vertex_prompt_tokens_details,
)
from litellm.llms.vertex_ai.common_utils import ( # noqa: E402
VertexAIError,
@ -41,6 +41,22 @@ ENDPOINT_INPUT_FILE = (
)
def test_vertex_prompt_tokens_details_rejects_malformed_details():
assert vertex_prompt_tokens_details({"promptTokensDetails": [1]}) is None
assert vertex_prompt_tokens_details({"promptTokensDetails": [{"modality": "AUDIO"}]}) is None
assert (
vertex_prompt_tokens_details(
{
"promptTokensDetails": [
{"modality": "AUDIO", "tokenCount": 1},
"malformed",
]
}
)
is None
)
# =========================================================================== #
# transform_openai_batch_request_to_vertex_ai_batch_request
# =========================================================================== #

184
tests/test_litellm/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py
View file

@ -10,6 +10,7 @@ Covers:
import pytest
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import (
_build_part_for_input,
@ -22,11 +23,19 @@ from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation
from litellm.types.llms.vertex_ai import VertexAIBatchEmbeddingsResponseObject
from litellm.types.utils import EmbeddingResponse
IMAGE_DATA_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
GCS_URL = "gs://my-bucket/image.png"
@pytest.fixture(autouse=True)
def _local_model_cost_map(monkeypatch):
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
class TestIsMultimodalInput:
def test_text_only_string(self):
assert _is_multimodal_input("hello world") is False
@ -324,7 +333,7 @@ class TestProcessEmbedContentResponseUsage:
)
assert result.usage.prompt_tokens == 258
assert result.usage.total_tokens == 258
assert result.usage.prompt_tokens_details.image_count == 1
assert result.usage.prompt_tokens_details.image_tokens == 258
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
@ -358,7 +367,7 @@ class TestProcessEmbedContentResponseUsage:
)
assert prompt_cost > 0
def test_video_modality_derives_seconds_and_text_floor(self):
def test_video_modality_preserves_token_count(self):
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
@ -374,10 +383,8 @@ class TestProcessEmbedContentResponseUsage:
response_json=response_json,
)
assert result.usage.prompt_tokens == 516
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.text_tokens == 1
assert result.usage.prompt_tokens_details.video_tokens == 516
assert result.usage.prompt_tokens_details.text_tokens == 0
def test_missing_usage_metadata_does_not_estimate_from_base64(self):
response_json = {"embedding": {"values": [0.1, 0.2]}}
@ -400,8 +407,7 @@ class TestProcessEmbedContentResponseUsage:
)
assert result.usage.prompt_tokens > 0
def test_file_reference_image_billed_per_image_not_text(self):
"""files/... image refs must bill per-image, not at the text token rate."""
def test_file_reference_image_billed_per_image_token_rate(self):
response_json = {
"embedding": {"values": [0.1, 0.2, 0.3]},
"usageMetadata": {
@ -422,7 +428,7 @@ class TestProcessEmbedContentResponseUsage:
}
},
)
assert result.usage.prompt_tokens_details.image_count == 1
assert result.usage.prompt_tokens_details.image_tokens == 258
assert result.usage.prompt_tokens_details.text_tokens == 0
prompt_cost, _ = generic_cost_per_token(
@ -430,10 +436,10 @@ class TestProcessEmbedContentResponseUsage:
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(0.00012)
assert prompt_cost == pytest.approx(258 * 4.5e-7)
def test_file_reference_non_image_not_counted_as_image(self):
"""A files/... ref resolving to a non-image mime must not be image-counted."""
"""A files/... ref resolving to a non-image mime keeps audio token billing."""
response_json = {
"embedding": {"values": [0.1, 0.2]},
"usageMetadata": {
@ -454,21 +460,18 @@ class TestProcessEmbedContentResponseUsage:
}
},
)
assert result.usage.prompt_tokens_details.image_count == 0
assert result.usage.prompt_tokens_details.audio_tokens == 64
assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.image_tokens == 0
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(2.0 * 0.00016)
assert prompt_cost == pytest.approx(64 * 6.5e-6)
def test_video_plus_audio_does_not_double_bill_text(self):
"""Video+audio responses must not get video tokens reassigned to text."""
"""Video and audio responses are billed from their respective token counts."""
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
@ -486,18 +489,145 @@ class TestProcessEmbedContentResponseUsage:
model=self.MODEL,
response_json=response_json,
)
assert result.usage.prompt_tokens_details.text_tokens == 1
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.audio_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.text_tokens == 0
assert result.usage.prompt_tokens_details.video_tokens == 516
assert result.usage.prompt_tokens_details.audio_tokens == 64
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
# 1 floor text token at 2e-7 + 2s of video at 7.9e-4 + 2s of audio at 1.6e-4
assert prompt_cost == pytest.approx(1 * 2e-7 + 2 * 0.00079 + 2 * 0.00016)
assert prompt_cost == pytest.approx(516 * 1.2e-5 + 64 * 6.5e-6)
def test_preview_alias_bills_audio_per_token(self):
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
"promptTokenCount": 64,
"totalTokenCount": 64,
"promptTokensDetails": [{"modality": "AUDIO", "tokenCount": 64}],
},
}
result = process_embed_content_response(
input="audio",
model_response=EmbeddingResponse(),
model="gemini-embedding-2-preview",
response_json=response_json,
)
prompt_cost, _ = generic_cost_per_token(
model="gemini-embedding-2-preview",
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(64 * 6.5e-6)
def test_image_without_modality_details_uses_image_rate(self):
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
"promptTokenCount": 258,
"totalTokenCount": 258,
},
}
result = process_embed_content_response(
input=IMAGE_DATA_URI,
model_response=EmbeddingResponse(),
model=self.MODEL,
response_json=response_json,
)
assert result.usage.prompt_tokens_details.image_tokens == 258
assert result.usage.prompt_tokens_details.text_tokens == 0
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(258 * 4.5e-7)
@pytest.mark.parametrize(
"input_value,resolved_files,expected_image_tokens",
[
(GCS_URL, {}, 258),
("gs://my-bucket/clip.mp4", {}, 0),
("gs://my-bucket/unknown.bin", {}, 0),
("files/image-123", {"files/image-123": {"mime_type": "image/jpeg"}}, 258),
("files/missing", {}, 0),
("data:application/octet-stream;base64,abc", {}, 0),
([[IMAGE_DATA_URI]], {}, 258),
([], {}, 0),
],
)
def test_missing_modality_details_classifies_image_inputs(self, input_value, resolved_files, expected_image_tokens):
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
"promptTokenCount": 258,
"totalTokenCount": 258,
},
}
result = process_embed_content_response(
input=input_value,
model_response=EmbeddingResponse(),
model=self.MODEL,
response_json=response_json,
resolved_files=resolved_files,
)
assert result.usage.prompt_tokens_details.image_tokens == expected_image_tokens
assert result.usage.prompt_tokens_details.text_tokens == 0
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
expected_rate = 4.5e-7 if expected_image_tokens else 2e-7
assert prompt_cost == pytest.approx(258 * expected_rate)
def test_mixed_text_and_image_without_modality_details_not_billed_as_image(self):
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
"promptTokenCount": 270,
"totalTokenCount": 270,
},
}
result = process_embed_content_response(
input=["a short caption", IMAGE_DATA_URI],
model_response=EmbeddingResponse(),
model=self.MODEL,
response_json=response_json,
)
assert result.usage.prompt_tokens_details.image_tokens == 0
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(270 * 2e-7)
def test_text_without_modality_details_uses_text_rate(self):
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
"promptTokenCount": 12,
"totalTokenCount": 12,
},
}
result = process_embed_content_response(
input="a short caption",
model_response=EmbeddingResponse(),
model=self.MODEL,
response_json=response_json,
)
assert result.usage.prompt_tokens_details.text_tokens == 0
assert result.usage.prompt_tokens_details.image_tokens == 0
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
usage=result.usage,
custom_llm_provider="vertex_ai",
)
assert prompt_cost == pytest.approx(12 * 2e-7)

51
tests/test_litellm/test_cost_calculator.py
View file

@ -3909,6 +3909,57 @@ def _batch_cache_usage() -> Usage:
)
def test_batch_cost_calculator_prices_multimodal_tokens_at_modality_rates():
from litellm.cost_calculator import batch_cost_calculator
model_info: ModelInfo = {
"input_cost_per_token_batches": 1e-7,
"input_cost_per_audio_token_batches": 3.25e-6,
"input_cost_per_image_token_batches": 2.25e-7,
"input_cost_per_video_token_batches": 6e-6,
}
usage = Usage(
prompt_tokens=100,
completion_tokens=0,
total_tokens=100,
prompt_tokens_details=PromptTokensDetailsWrapper(
audio_tokens=64,
image_tokens=10,
video_tokens=6,
),
)
prompt_cost, _ = batch_cost_calculator(
usage=usage,
model="gemini-embedding-2",
custom_llm_provider="vertex_ai",
model_info=model_info,
)
assert prompt_cost == pytest.approx(20 * 1e-7 + 64 * 3.25e-6 + 10 * 2.25e-7 + 6 * 6e-6)
def test_batch_cost_calculator_falls_back_to_text_batch_rate_for_modalities():
from litellm.cost_calculator import batch_cost_calculator
model_info: ModelInfo = {"input_cost_per_token_batches": 1e-7}
usage = Usage(
prompt_tokens=100,
completion_tokens=0,
total_tokens=100,
prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=64),
)
prompt_cost, _ = batch_cost_calculator(
usage=usage,
model="gemini-embedding-2",
custom_llm_provider="vertex_ai",
model_info=model_info,
)
assert prompt_cost == pytest.approx(100 * 1e-7)
def test_batch_cost_calculator_prices_cache_creation_tokens_at_cache_write_rate():
"""
LIT-4008 regression: anthropic batch usage is dominated by cache tokens.

20
tests/test_litellm/test_utils.py
View file

@ -892,7 +892,10 @@ def validate_model_cost_values(model_data, exceptions=None):
"input_cost_per_video_per_second_above_8s_interval",
"input_cost_per_video_per_second_above_15s_interval",
"input_cost_per_video_per_second_above_128k_tokens",
"input_cost_per_audio_token_batches",
"input_cost_per_image_token_batches",
"input_cost_per_token_batches",
"input_cost_per_video_token_batches",
"output_cost_per_token_batches",
"input_cost_per_token_cache_hit",
"cache_creation_input_token_cost",
@ -1041,7 +1044,10 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"input_cost_per_second": {"type": "number"},
"input_cost_per_token": {"type": "number"},
"input_cost_per_token_above_128k_tokens": {"type": "number"},
"input_cost_per_audio_token_batches": {"type": "number"},
"input_cost_per_image_token_batches": {"type": "number"},
"input_cost_per_token_batches": {"type": "number"},
"input_cost_per_video_token_batches": {"type": "number"},
"input_cost_per_token_cache_hit": {"type": "number"},
"input_cost_per_video_per_second": {"type": "number"},
"input_cost_per_video_per_second_above_8s_interval": {"type": "number"},
@ -2946,7 +2952,7 @@ def test_model_info_for_openrouter_kimi_k2_5():
def test_gemini_embedding_2_ga_in_cost_map():
"""GA and Vertex preview gemini-embedding-2 entries align with multimodal unit pricing."""
"""GA and Vertex preview gemini-embedding-2 entries align with multimodal token pricing."""
import json
from pathlib import Path
@ -2968,9 +2974,15 @@ def test_gemini_embedding_2_ga_in_cost_map():
assert info.get("mode") == "embedding"
assert info.get("supports_multimodal") is True
assert info.get("input_cost_per_token") == 2e-07
assert info.get("input_cost_per_image") == 0.00012
assert info.get("input_cost_per_audio_per_second") == 0.00016
assert info.get("input_cost_per_video_per_second") == 0.00079
assert info.get("input_cost_per_audio_token") == 6.5e-06
assert info.get("input_cost_per_image_token") == 4.5e-07
assert info.get("input_cost_per_video_token") == 1.2e-05
assert info.get("input_cost_per_audio_token_batches") == 3.25e-06
assert info.get("input_cost_per_image_token_batches") == 2.25e-07
assert info.get("input_cost_per_video_token_batches") == 6e-06
assert "input_cost_per_image" not in info
assert "input_cost_per_audio_per_second" not in info
assert "input_cost_per_video_per_second" not in info
if provider in ("vertex_ai-embedding-models", "vertex_ai"):
assert (
info.get("uses_embed_content") is True

12
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -29857,6 +29857,8 @@ export interface components {
input_cost_per_audio_per_second_above_128k_tokens?: number | null;
/** Input Cost Per Audio Token */
input_cost_per_audio_token?: number | null;
/** Input Cost Per Audio Token Batches */
input_cost_per_audio_token_batches?: number | null;
/** Input Cost Per Character */
input_cost_per_character?: number | null;
/** Input Cost Per Character Above 128K Tokens */
@ -29867,6 +29869,8 @@ export interface components {
input_cost_per_image_above_128k_tokens?: number | null;
/** Input Cost Per Image Token */
input_cost_per_image_token?: number | null;
/** Input Cost Per Image Token Batches */
input_cost_per_image_token_batches?: number | null;
/** Input Cost Per Pixel */
input_cost_per_pixel?: number | null;
/** Input Cost Per Query */
@ -29909,6 +29913,8 @@ export interface components {
input_cost_per_video_per_second_above_8s_interval?: number | null;
/** Input Cost Per Video Token */
input_cost_per_video_token?: number | null;
/** Input Cost Per Video Token Batches */
input_cost_per_video_token_batches?: number | null;
/** Itpm */
itpm?: number | null;
/** Keepalive Seconds */
@ -40077,6 +40083,8 @@ export interface components {
input_cost_per_audio_per_second_above_128k_tokens?: number | null;
/** Input Cost Per Audio Token */
input_cost_per_audio_token?: number | null;
/** Input Cost Per Audio Token Batches */
input_cost_per_audio_token_batches?: number | null;
/** Input Cost Per Character */
input_cost_per_character?: number | null;
/** Input Cost Per Character Above 128K Tokens */
@ -40087,6 +40095,8 @@ export interface components {
input_cost_per_image_above_128k_tokens?: number | null;
/** Input Cost Per Image Token */
input_cost_per_image_token?: number | null;
/** Input Cost Per Image Token Batches */
input_cost_per_image_token_batches?: number | null;
/** Input Cost Per Pixel */
input_cost_per_pixel?: number | null;
/** Input Cost Per Query */
@ -40129,6 +40139,8 @@ export interface components {
input_cost_per_video_per_second_above_8s_interval?: number | null;
/** Input Cost Per Video Token */
input_cost_per_video_token?: number | null;
/** Input Cost Per Video Token Batches */
input_cost_per_video_token_batches?: number | null;
/** Itpm */
itpm?: number | null;
/** Keepalive Seconds */