scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-15 23:31:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 165

fix(cost): bill batch embeddings per modality token rate

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-15 01:50:21 +00:00
parent a28ea22ec1
commit ce83fac351
9 changed files with 146 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
ci_cd/generate_model_prices_schema.py
View file

@ -172,7 +172,10 @@ COST_DESCRIPTIONS: dict[str, str] = {
),
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
"input_cost_per_audio_token_batches": "USD per audio prompt token via the provider's batch API.",
"input_cost_per_image_token_batches": "USD per image prompt token via the provider's batch API.",
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
"input_cost_per_video_token_batches": "USD per video prompt token via the provider's batch API.",
"output_cost_per_token_batches": "USD per generated token via the provider's batch API.",
}

30
litellm/cost_calculator.py
View file

@ -2337,7 +2337,35 @@ def batch_cost_calculator(
total_prompt_cost = 0.0
total_completion_cost = 0.0
if input_cost_per_token_batches is not None:
total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
batch_details: Final = parse_prompt_tokens_details(usage)
audio_tokens, image_tokens, video_tokens = (
batch_details["audio_tokens"],
batch_details["image_tokens"],
batch_details["video_tokens"],
)
modality_rates: Final = (
cast(float, model_info.get("input_cost_per_audio_token_batches"))
if model_info.get("input_cost_per_audio_token_batches") is not None
else input_cost_per_token_batches,
cast(float, model_info.get("input_cost_per_image_token_batches"))
if model_info.get("input_cost_per_image_token_batches") is not None
else input_cost_per_token_batches,
cast(float, model_info.get("input_cost_per_video_token_batches"))
if model_info.get("input_cost_per_video_token_batches") is not None
else input_cost_per_token_batches,
)
total_prompt_cost = sum(
tokens * rate
for tokens, rate in zip(
(
max(cast(int, usage.prompt_tokens) - audio_tokens - image_tokens - video_tokens, 0),
audio_tokens,
image_tokens,
video_tokens,
),
(input_cost_per_token_batches, *modality_rates),
)
)
elif input_cost_per_token:
details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens: Final = details["cache_hit_tokens"]

18
litellm/model_prices_and_context_window_backup.json
View file

@ -25602,10 +25602,13 @@
},
"gemini-embedding-2-preview": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25617,10 +25620,13 @@
},
"gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25633,10 +25639,13 @@
},
"vertex_ai/gemini-embedding-2-preview": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25649,10 +25658,13 @@
},
"vertex_ai/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25692,10 +25704,13 @@
"gemini/gemini-embedding-2-preview": {
"deprecation_date": "2026-08-10",
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25709,10 +25724,13 @@
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,

6
litellm/types/utils.py
View file

@ -283,8 +283,11 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_video_token: float | None # for gemini omni models with video input
input_cost_per_audio_per_second: float | None # only for vertex ai models
input_cost_per_video_per_second: float | None # only for vertex ai models
input_cost_per_audio_token_batches: ReadOnly[float | None]
input_cost_per_image_token_batches: ReadOnly[float | None]
input_cost_per_second: float | None # for OpenAI Speech models
input_cost_per_token_batches: float | None
input_cost_per_video_token_batches: ReadOnly[float | None]
output_cost_per_token_batches: float | None
output_cost_per_token: Required[float | None]
output_cost_per_token_flex: float | None # OpenAI flex service tier pricing
@ -3583,7 +3586,10 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
input_cost_per_video_per_second_above_128k_tokens: float | None = None
input_cost_per_video_per_second_above_15s_interval: float | None = None
input_cost_per_video_per_second_above_8s_interval: float | None = None
input_cost_per_audio_token_batches: float | None = None
input_cost_per_image_token_batches: float | None = None
input_cost_per_token_batches: float | None = None
input_cost_per_video_token_batches: float | None = None
output_cost_per_token_batches: float | None = None
output_cost_per_token_flex: float | None = None
output_cost_per_token_priority: float | None = None

3
litellm/utils.py
View file

@ -5923,10 +5923,13 @@ def _get_model_info_helper(
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None),
input_cost_per_image_token_batches=_model_info.get("input_cost_per_image_token_batches", None),
input_cost_per_image=_model_info.get("input_cost_per_image", None),
input_cost_per_audio_per_second=_model_info.get("input_cost_per_audio_per_second", None),
input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None),
input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"),
input_cost_per_video_token_batches=_model_info.get("input_cost_per_video_token_batches", None),
output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"),
output_cost_per_token=_output_cost_per_token,
output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None),

18
model_prices_and_context_window.json
View file

@ -25602,10 +25602,13 @@
},
"gemini-embedding-2-preview": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25617,10 +25620,13 @@
},
"gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai-embedding-models",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25633,10 +25639,13 @@
},
"vertex_ai/gemini-embedding-2-preview": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25649,10 +25658,13 @@
},
"vertex_ai/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "vertex_ai",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25692,10 +25704,13 @@
"gemini/gemini-embedding-2-preview": {
"deprecation_date": "2026-08-10",
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,
@ -25709,10 +25724,13 @@
},
"gemini/gemini-embedding-2": {
"input_cost_per_audio_token": 6.5e-06,
"input_cost_per_audio_token_batches": 3.25e-06,
"input_cost_per_image_token": 4.5e-07,
"input_cost_per_image_token_batches": 2.25e-07,
"input_cost_per_token": 2e-07,
"input_cost_per_token_batches": 1e-07,
"input_cost_per_video_token": 1.2e-05,
"input_cost_per_video_token_batches": 6e-06,
"litellm_provider": "gemini",
"max_input_tokens": 8192,
"max_tokens": 8192,

15
model_prices_and_context_window.schema.json
View file

@ -249,6 +249,11 @@
"type": "number",
"minimum": 0
},
"input_cost_per_audio_token_batches": {
"type": "number",
"minimum": 0,
"description": "USD per audio prompt token via the provider's batch API."
},
"input_cost_per_audio_token_priority": {
"type": "number",
"minimum": 0,
@ -276,6 +281,11 @@
"type": "number",
"minimum": 0
},
"input_cost_per_image_token_batches": {
"type": "number",
"minimum": 0,
"description": "USD per image prompt token via the provider's batch API."
},
"input_cost_per_pixel": {
"type": "number",
"minimum": 0
@ -379,6 +389,11 @@
"type": "number",
"minimum": 0
},
"input_cost_per_video_token_batches": {
"type": "number",
"minimum": 0,
"description": "USD per video prompt token via the provider's batch API."
},
"input_dbu_cost_per_token": {
"type": "number",
"minimum": 0

51
tests/test_litellm/test_cost_calculator.py
View file

@ -3909,6 +3909,57 @@ def _batch_cache_usage() -> Usage:
)
def test_batch_cost_calculator_prices_multimodal_tokens_at_modality_rates():
from litellm.cost_calculator import batch_cost_calculator
model_info: ModelInfo = {
"input_cost_per_token_batches": 1e-7,
"input_cost_per_audio_token_batches": 3.25e-6,
"input_cost_per_image_token_batches": 2.25e-7,
"input_cost_per_video_token_batches": 6e-6,
}
usage = Usage(
prompt_tokens=100,
completion_tokens=0,
total_tokens=100,
prompt_tokens_details=PromptTokensDetailsWrapper(
audio_tokens=64,
image_tokens=10,
video_tokens=6,
),
)
prompt_cost, _ = batch_cost_calculator(
usage=usage,
model="gemini-embedding-2",
custom_llm_provider="vertex_ai",
model_info=model_info,
)
assert prompt_cost == pytest.approx(20 * 1e-7 + 64 * 3.25e-6 + 10 * 2.25e-7 + 6 * 6e-6)
def test_batch_cost_calculator_falls_back_to_text_batch_rate_for_modalities():
from litellm.cost_calculator import batch_cost_calculator
model_info: ModelInfo = {"input_cost_per_token_batches": 1e-7}
usage = Usage(
prompt_tokens=100,
completion_tokens=0,
total_tokens=100,
prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=64),
)
prompt_cost, _ = batch_cost_calculator(
usage=usage,
model="gemini-embedding-2",
custom_llm_provider="vertex_ai",
model_info=model_info,
)
assert prompt_cost == pytest.approx(100 * 1e-7)
def test_batch_cost_calculator_prices_cache_creation_tokens_at_cache_write_rate():
"""
LIT-4008 regression: anthropic batch usage is dominated by cache tokens.

3
tests/test_litellm/test_utils.py
View file

@ -2971,6 +2971,9 @@ def test_gemini_embedding_2_ga_in_cost_map():
assert info.get("input_cost_per_audio_token") == 6.5e-06
assert info.get("input_cost_per_image_token") == 4.5e-07
assert info.get("input_cost_per_video_token") == 1.2e-05
assert info.get("input_cost_per_audio_token_batches") == 3.25e-06
assert info.get("input_cost_per_image_token_batches") == 2.25e-07
assert info.get("input_cost_per_video_token_batches") == 6e-06
assert "input_cost_per_image" not in info
assert "input_cost_per_audio_per_second" not in info
assert "input_cost_per_video_per_second" not in info