fix(cost): avoid double billing Gemini embedding modalities

This commit is contained in:
Tin Chi Lo 2026-09-14 17:23:38 -07:00
parent d1fc231545
commit c698972020
7 changed files with 171 additions and 89 deletions

View file

@ -707,6 +707,24 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value:
return 0.0
def _calculate_modality_input_cost(
model_info: ModelInfo,
tokens: int,
token_cost_key: str,
units: float,
unit_cost_key: str,
token_fallback_key: str | None = None,
cached_tokens: int = 0,
) -> float:
token_rate: Final = _get_cost_per_unit(model_info, token_cost_key, None)
if token_rate is not None and tokens + cached_tokens > 0:
return tokens * token_rate
unit_rate: Final = _get_cost_per_unit(model_info, unit_cost_key, None)
if unit_rate is not None and units > 0:
return units * unit_rate
return calculate_cost_component(model_info, token_fallback_key or token_cost_key, tokens)
def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None:
# Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
cost_per_unit: Final = model_info.get(cost_key)
@ -955,26 +973,30 @@ def _calculate_input_cost(
audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost
)
### AUDIO COST
if prompt_tokens_details["audio_tokens"]:
audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
### IMAGE TOKEN COST
if prompt_tokens_details["image_tokens"]:
# For image token costs:
# First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
image_token_cost_key = "input_cost_per_image_token"
if model_info.get(image_token_cost_key) is None:
image_token_cost_key = "input_cost_per_token"
prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
### VIDEO TOKEN COST
if prompt_tokens_details["video_tokens"]:
video_token_cost_key = "input_cost_per_video_token"
if model_info.get(video_token_cost_key) is None:
video_token_cost_key = "input_cost_per_token"
prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"])
prompt_cost += _calculate_modality_input_cost(
model_info,
prompt_tokens_details["audio_tokens"],
_get_service_tier_cost_key("input_cost_per_audio_token", service_tier),
prompt_tokens_details["audio_length_seconds"],
"input_cost_per_audio_per_second",
cached_tokens=cache_hit_audio_tokens,
)
prompt_cost += _calculate_modality_input_cost(
model_info,
prompt_tokens_details["image_tokens"],
"input_cost_per_image_token",
prompt_tokens_details["image_count"],
"input_cost_per_image",
token_fallback_key="input_cost_per_token",
)
prompt_cost += _calculate_modality_input_cost(
model_info,
prompt_tokens_details["video_tokens"],
"input_cost_per_video_token",
prompt_tokens_details["video_length_seconds"],
"input_cost_per_video_per_second",
token_fallback_key="input_cost_per_token",
)
### CACHE WRITING COST - Now uses tiered pricing
if (
@ -996,28 +1018,6 @@ def _calculate_input_cost(
prompt_tokens_details["character_count"],
)
### IMAGE COUNT COST
if prompt_tokens_details["image_count"]:
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image", prompt_tokens_details["image_count"]
)
### VIDEO LENGTH COST
if prompt_tokens_details["video_length_seconds"]:
prompt_cost += calculate_cost_component(
model_info,
"input_cost_per_video_per_second",
prompt_tokens_details["video_length_seconds"],
)
### AUDIO LENGTH COST
if prompt_tokens_details["audio_length_seconds"]:
prompt_cost += calculate_cost_component(
model_info,
"input_cost_per_audio_per_second",
prompt_tokens_details["audio_length_seconds"],
)
if prompt_tokens_details["query_count"]:
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_query", prompt_tokens_details["query_count"]

View file

@ -374,25 +374,21 @@ def _usage_from_embed_content_response(
details: Final[Sequence[PromptTokensDetails]] = usage_metadata.get("promptTokensDetails") or ()
text_tokens: Final = _tokens_for_modality(details, "TEXT")
audio_tokens: Final = _tokens_for_modality(details, "AUDIO")
image_tokens: Final = _tokens_for_modality(details, "IMAGE")
video_tokens: Final = _tokens_for_modality(details, "VIDEO")
image_count: Final = _count_input_images(input, resolved_files)
video_length_seconds: Final = video_tokens / _VIDEO_TOKENS_PER_SECOND if video_tokens > 0 else 0.0
audio_length_seconds: Final = audio_tokens / _AUDIO_TOKENS_PER_SECOND if audio_tokens > 0 else 0.0
# generic_cost_per_token rewrites text_tokens to the full prompt minus
# other modalities when both text_tokens and image_count are zero. For
# video, that misallocates video tokens to text; a 1-token floor sidesteps
# the rewrite and keeps billing on input_cost_per_video_per_second.
needs_video_text_floor: Final = video_length_seconds > 0 and text_tokens == 0 and image_count == 0
resolved_text_tokens: Final = 1 if needs_video_text_floor else text_tokens
return Usage(
prompt_tokens=prompt_tokens,
total_tokens=total_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=resolved_text_tokens,
text_tokens=text_tokens,
audio_tokens=audio_tokens,
image_tokens=image_tokens,
video_tokens=video_tokens,
image_count=image_count,
video_length_seconds=video_length_seconds,
audio_length_seconds=audio_length_seconds,

View file

@ -25602,9 +25602,12 @@
},
"gemini-embedding-2-preview": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25622,6 +25625,7 @@
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25634,9 +25638,12 @@
},
"vertex_ai/gemini-embedding-2-preview": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25655,6 +25662,7 @@
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25694,9 +25702,12 @@
"gemini/gemini-embedding-2-preview": {
"deprecation_date": "2026-08-10",
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25710,9 +25721,12 @@
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,

View file

@ -25602,9 +25602,12 @@
},
"gemini-embedding-2-preview": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25622,6 +25625,7 @@
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25634,9 +25638,12 @@
},
"vertex_ai/gemini-embedding-2-preview": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25655,6 +25662,7 @@
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25694,9 +25702,12 @@
"gemini/gemini-embedding-2-preview": {
"deprecation_date": "2026-08-10",
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25710,9 +25721,12 @@
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_per_second": 0.00016,
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_image": 0.00012,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_video_per_second": 0.00079,
"input_cost_per_video_token": 1.2e-05,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,

View file

@ -375,6 +375,10 @@
"minimum": 0,
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
},
"input_cost_per_video_token": {
"type": "number",
"minimum": 0
},
"input_dbu_cost_per_token": {
"type": "number",
"minimum": 0

View file

@ -1,5 +1,6 @@
import json
from datetime import datetime, timezone
from typing import Final
import pytest
from fastapi.testclient import TestClient
@ -5334,3 +5335,44 @@ def test_realtime_models_bill_cached_text_and_audio_at_their_cache_read_rates(
prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider=custom_llm_provider)
assert prompt_cost == pytest.approx(expected_prompt_cost)
@pytest.mark.parametrize("tokens,cached_audio,token_rate,unit_rate,expected", [
(10, 0, 0.1, 7.0, 3.0),
(10, 0, 0.0, 7.0, 0.0),
(10, 0, None, 7.0, 42.0),
(0, 0, 0.1, 7.0, 42.0),
(0, 10, 0.1, 7.0, 28.0),
(10, 0, None, None, 0.2),
(10, 0, None, 0.0, 0.0),
])
def test_modality_tokens_and_physical_units_are_alternative_prices(
tokens: int, cached_audio: int, token_rate: float | None,
unit_rate: float | None, expected: float,
) -> None:
info: Final[ModelInfo] = {
"input_cost_per_token": 0.01,
"input_cost_per_audio_token": token_rate,
"input_cost_per_image_token": token_rate,
"input_cost_per_video_token": token_rate,
"input_cost_per_audio_per_second": unit_rate,
"input_cost_per_image": unit_rate,
"input_cost_per_video_per_second": unit_rate,
"cache_read_input_token_cost": 0.0,
}
usage: Final = Usage(
prompt_tokens=3 * tokens + cached_audio,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=0, audio_tokens=tokens + cached_audio,
image_tokens=tokens, video_tokens=tokens,
cached_tokens=cached_audio,
cached_tokens_details={"audio_tokens": cached_audio},
image_count=2, audio_length_seconds=2.0,
video_length_seconds=2.0,
),
)
prompt_cost, completion_cost = generic_cost_per_token(
"modality-unit-test", usage, "vertex_ai", model_info=info,
)
assert prompt_cost == pytest.approx(expected)
assert completion_cost == 0

View file

@ -299,15 +299,25 @@ class TestProcessResponse:
)
@pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("model,provider", [
("gemini-embedding-2", "vertex_ai"),
("gemini-embedding-2-preview", "vertex_ai"),
("vertex_ai/gemini-embedding-2", "vertex_ai"),
("vertex_ai/gemini-embedding-2-preview", "vertex_ai"),
("gemini-embedding-2", "gemini"),
("gemini-embedding-2-preview", "gemini"),
])
class TestProcessEmbedContentResponseUsage:
"""Gemini Embedding 2 embedContent usageMetadata must drive spend.
Regression for multimodal calls recording prompt_tokens=0 / spend=$0.
"""
MODEL = "gemini-embedding-2"
def test_multimodal_image_preserves_usage_metadata(self):
@pytest.mark.parametrize("inputs,image_count", [(IMAGE_DATA_URI, 1), ([], 0)])
def test_multimodal_image_preserves_usage_metadata(
self, model: str, provider: str, inputs: str | list[str], image_count: int,
) -> None:
response_json = {
"embedding": {"values": [0.1, 0.2, 0.3]},
"usageMetadata": {
@ -317,23 +327,24 @@ class TestProcessEmbedContentResponseUsage:
},
}
result = process_embed_content_response(
input=[IMAGE_DATA_URI],
input=inputs,
model_response=EmbeddingResponse(),
model=self.MODEL,
model=model,
response_json=response_json,
)
assert result.usage.prompt_tokens == 258
assert result.usage.total_tokens == 258
assert result.usage.prompt_tokens_details.image_count == 1
assert result.usage.prompt_tokens_details.image_count == image_count
assert result.usage.prompt_tokens_details.image_tokens == 258
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
model=model,
usage=result.usage,
custom_llm_provider="vertex_ai",
custom_llm_provider=provider,
)
assert prompt_cost > 0
assert prompt_cost == pytest.approx(258 * 4.5e-7)
def test_text_modality_detail_populated(self):
def test_text_modality_detail_populated(self, model: str, provider: str) -> None:
response_json = {
"embedding": {"values": [0.1, 0.2]},
"usageMetadata": {
@ -345,20 +356,20 @@ class TestProcessEmbedContentResponseUsage:
result = process_embed_content_response(
input="a short caption",
model_response=EmbeddingResponse(),
model=self.MODEL,
model=model,
response_json=response_json,
)
assert result.usage.prompt_tokens == 12
assert result.usage.prompt_tokens_details.text_tokens == 12
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
model=model,
usage=result.usage,
custom_llm_provider="vertex_ai",
custom_llm_provider=provider,
)
assert prompt_cost > 0
assert prompt_cost == pytest.approx(12 * 2e-7)
def test_video_modality_derives_seconds_and_text_floor(self):
def test_video_modality_preserves_tokens_without_text(self, model: str, provider: str) -> None:
response_json = {
"embedding": {"values": [0.1]},
"usageMetadata": {
@ -370,38 +381,38 @@ class TestProcessEmbedContentResponseUsage:
result = process_embed_content_response(
input=["gs://bucket/clip.mp4"],
model_response=EmbeddingResponse(),
model=self.MODEL,
model=model,
response_json=response_json,
)
assert result.usage.prompt_tokens == 516
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(
2.0
)
assert result.usage.prompt_tokens_details.text_tokens == 1
assert result.usage.prompt_tokens_details.video_tokens == 516
assert result.usage.prompt_tokens_details.text_tokens == 0
def test_missing_usage_metadata_does_not_estimate_from_base64(self):
def test_missing_usage_metadata_does_not_estimate_from_base64(self, model: str, provider: str) -> None:
response_json = {"embedding": {"values": [0.1, 0.2]}}
result = process_embed_content_response(
input=[IMAGE_DATA_URI],
model_response=EmbeddingResponse(),
model=self.MODEL,
model=model,
response_json=response_json,
)
assert result.usage.prompt_tokens == 0
assert result.usage.total_tokens == 0
def test_missing_usage_metadata_text_falls_back_to_token_counter(self):
def test_missing_usage_metadata_text_falls_back_to_token_counter(self, model: str, provider: str) -> None:
response_json = {"embedding": {"values": [0.1, 0.2]}}
result = process_embed_content_response(
input="hello world this is plain text",
model_response=EmbeddingResponse(),
model=self.MODEL,
model=model,
response_json=response_json,
)
assert result.usage.prompt_tokens > 0
def test_file_reference_image_billed_per_image_not_text(self):
"""files/... image refs must bill per-image, not at the text token rate."""
def test_file_reference_image_billed_as_image_tokens(self, model: str, provider: str) -> None:
response_json = {
"embedding": {"values": [0.1, 0.2, 0.3]},
"usageMetadata": {
@ -413,7 +424,7 @@ class TestProcessEmbedContentResponseUsage:
result = process_embed_content_response(
input=["files/img123"],
model_response=EmbeddingResponse(),
model=self.MODEL,
model=model,
response_json=response_json,
resolved_files={
"files/img123": {
@ -426,13 +437,14 @@ class TestProcessEmbedContentResponseUsage:
assert result.usage.prompt_tokens_details.text_tokens == 0
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
model=model,
usage=result.usage,
custom_llm_provider="vertex_ai",
custom_llm_provider=provider,
)
assert prompt_cost == pytest.approx(0.00012)
assert result.usage.prompt_tokens_details.image_tokens == 258
assert prompt_cost == pytest.approx(258 * 4.5e-7)
def test_file_reference_non_image_not_counted_as_image(self):
def test_file_reference_non_image_not_counted_as_image(self, model: str, provider: str) -> None:
"""A files/... ref resolving to a non-image mime must not be image-counted."""
response_json = {
"embedding": {"values": [0.1, 0.2]},
@ -445,7 +457,7 @@ class TestProcessEmbedContentResponseUsage:
result = process_embed_content_response(
input=["files/clip1"],
model_response=EmbeddingResponse(),
model=self.MODEL,
model=model,
response_json=response_json,
resolved_files={
"files/clip1": {
@ -461,13 +473,13 @@ class TestProcessEmbedContentResponseUsage:
)
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
model=model,
usage=result.usage,
custom_llm_provider="vertex_ai",
custom_llm_provider=provider,
)
assert prompt_cost == pytest.approx(2.0 * 0.00016)
assert prompt_cost == pytest.approx(64 * 6.5e-6)
def test_video_plus_audio_does_not_double_bill_text(self):
def test_video_plus_audio_does_not_double_bill_text(self, model: str, provider: str) -> None:
"""Video+audio responses must not get video tokens reassigned to text."""
response_json = {
"embedding": {"values": [0.1]},
@ -483,10 +495,11 @@ class TestProcessEmbedContentResponseUsage:
result = process_embed_content_response(
input=["gs://bucket/clip.mp4"],
model_response=EmbeddingResponse(),
model=self.MODEL,
model=model,
response_json=response_json,
)
assert result.usage.prompt_tokens_details.text_tokens == 1
assert result.usage.prompt_tokens_details.video_tokens == 516
assert result.usage.prompt_tokens_details.text_tokens == 0
assert result.usage.prompt_tokens_details.video_length_seconds == pytest.approx(
2.0
)
@ -495,9 +508,8 @@ class TestProcessEmbedContentResponseUsage:
)
prompt_cost, _ = generic_cost_per_token(
model=self.MODEL,
model=model,
usage=result.usage,
custom_llm_provider="vertex_ai",
custom_llm_provider=provider,
)
# 1 floor text token at 2e-7 + 2s of video at 7.9e-4 + 2s of audio at 1.6e-4
assert prompt_cost == pytest.approx(1 * 2e-7 + 2 * 0.00079 + 2 * 0.00016)
assert prompt_cost == pytest.approx(516 * 12e-6 + 64 * 6.5e-6)