mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
feat(cost): bill batch cached image input tokens for OpenAI image models (#50997)
* feat(cost): bill batch cached image input tokens for OpenAI image models * fix(cost): bill batch cached audio and image tokens without a text cache price * fix(cost): honor modality cache prices in the half-price batch fallback --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: kerry <kerry@berri.ai>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
f0d234c4c6
commit
263281ac89
11 changed files with 290 additions and 40 deletions
|
|
@ -28,6 +28,7 @@ pub struct ModelInfoFields {
|
|||
pub cache_read_input_audio_token_cost_flex: Option<f64>,
|
||||
pub cache_read_input_audio_token_cost_priority: Option<f64>,
|
||||
pub cache_read_input_audio_token_cost_batches: Option<f64>,
|
||||
pub cache_read_input_image_token_cost_batches: Option<f64>,
|
||||
pub explicit_cache_read_input_token_cost: Option<f64>,
|
||||
pub prompt_cache_min_tokens: Option<i64>,
|
||||
pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
|
||||
|
|
|
|||
|
|
@ -49,7 +49,6 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
BilledTokenRates,
|
||||
CostCalculatorUtils,
|
||||
calculate_cache_writing_cost,
|
||||
calculate_cost_component,
|
||||
generic_cost_per_character,
|
||||
generic_cost_per_token,
|
||||
get_batch_cost_rates,
|
||||
|
|
@ -3177,6 +3176,21 @@ def default_video_cost_calculator(
|
|||
return 0.0
|
||||
|
||||
|
||||
def _optional_batch_rate(
|
||||
model_info: ModelInfo,
|
||||
key: Literal[
|
||||
"input_cost_per_audio_token_batches",
|
||||
"cache_read_input_audio_token_cost_batches",
|
||||
"input_cost_per_image_token_batches",
|
||||
"input_cost_per_video_token_batches",
|
||||
],
|
||||
prompt_tokens: int,
|
||||
) -> float | None:
|
||||
context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens)
|
||||
context_rate: Final = get_cost_per_unit(model_info, context_key, None)
|
||||
return model_info.get(key) if context_rate is None else context_rate
|
||||
|
||||
|
||||
def _batch_rate(
|
||||
model_info: ModelInfo,
|
||||
key: Literal[
|
||||
|
|
@ -3189,9 +3203,7 @@ def _batch_rate(
|
|||
*,
|
||||
prompt_tokens: int = 0,
|
||||
) -> float:
|
||||
context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens)
|
||||
context_rate: Final = get_cost_per_unit(model_info, context_key, None)
|
||||
rate: Final = model_info.get(key) if context_rate is None else context_rate
|
||||
rate: Final = _optional_batch_rate(model_info, key, prompt_tokens)
|
||||
return fallback if rate is None else rate
|
||||
|
||||
|
||||
|
|
@ -3275,30 +3287,14 @@ def batch_cost_calculator(
|
|||
model_info,
|
||||
batch_rates.input,
|
||||
batch_rates.cache_read,
|
||||
batch_rates.cache_read_image,
|
||||
batch_rates.cache_creation,
|
||||
batch_rates.cache_creation_rates_by_ttl,
|
||||
)
|
||||
elif input_cost_per_token is not None:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cache_read_tokens: Final = details["cache_hit_tokens"]
|
||||
cache_creation_tokens: Final = details["cache_creation_tokens"]
|
||||
|
||||
# Subtract cached tokens from prompt_tokens before calculating cost
|
||||
# Fixes issue where cached tokens are being charged again
|
||||
base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens
|
||||
total_prompt_cost = (
|
||||
base_input_tokens * (input_cost_per_token) / 2
|
||||
) # batch cost is usually half of the regular token cost
|
||||
|
||||
# Add cache read cost if applicable
|
||||
cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None)
|
||||
total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2
|
||||
|
||||
configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost")
|
||||
cache_creation_cost: Final = (
|
||||
input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost
|
||||
total_prompt_cost = _half_price_prompt_cost(
|
||||
usage, model_info, input_cost_per_token, batch_rates.cache_read_image
|
||||
)
|
||||
total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
|
||||
text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token)
|
||||
image_rate: Final = _batch_or_half(
|
||||
model_info.get("output_cost_per_image_token_batches"),
|
||||
|
|
@ -3324,15 +3320,54 @@ def _standard_rate(model_info: ModelInfo, key: str, prompt_tokens: int) -> float
|
|||
return get_cost_per_unit(model_info, get_modality_token_cost_key(model_info, key, prompt_tokens), None)
|
||||
|
||||
|
||||
def _cached_audio_batch_fallback(
|
||||
model_info: ModelInfo, batch_cache_read_rate: float | None, cache_read_rate: float | None, prompt_tokens: int
|
||||
) -> float:
|
||||
def _cached_modality_rate(
|
||||
model_info: ModelInfo,
|
||||
standard_key: Literal["cache_read_input_audio_token_cost", "cache_read_input_image_token_cost"],
|
||||
batch_modality_rate: float | None,
|
||||
batch_cache_read_rate: float | None,
|
||||
cache_read_rate: float | None,
|
||||
prompt_tokens: int,
|
||||
) -> float | None:
|
||||
if batch_modality_rate is not None:
|
||||
return batch_modality_rate
|
||||
if batch_cache_read_rate is not None:
|
||||
return batch_cache_read_rate
|
||||
standard_audio_rate: Final = _standard_rate(model_info, "cache_read_input_audio_token_cost", prompt_tokens)
|
||||
if standard_audio_rate is not None:
|
||||
return standard_audio_rate
|
||||
return 0.0 if cache_read_rate is None else cache_read_rate
|
||||
standard_modality_rate: Final = _standard_rate(model_info, standard_key, prompt_tokens)
|
||||
return cache_read_rate if standard_modality_rate is None else standard_modality_rate
|
||||
|
||||
|
||||
def _half_price_prompt_cost(
|
||||
usage: Usage, model_info: ModelInfo, input_cost_per_token: float, batch_cache_read_image_rate: float | None
|
||||
) -> float:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cache_creation_tokens: Final = details["cache_creation_tokens"]
|
||||
base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens
|
||||
cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None)
|
||||
half_cache_read_rate: Final = (get_cost_per_unit(model_info, cache_read_cost_key, 0.0) or 0.0) / 2
|
||||
cached_audio_rate: Final = _batch_or_half(
|
||||
_optional_batch_rate(model_info, "cache_read_input_audio_token_cost_batches", usage.prompt_tokens),
|
||||
_standard_rate(model_info, "cache_read_input_audio_token_cost", usage.prompt_tokens),
|
||||
default=half_cache_read_rate,
|
||||
)
|
||||
cached_image_rate: Final = _batch_or_half(
|
||||
batch_cache_read_image_rate,
|
||||
_standard_rate(model_info, "cache_read_input_image_token_cost", usage.prompt_tokens),
|
||||
default=half_cache_read_rate,
|
||||
)
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"]
|
||||
cached_image_tokens: Final = details["cached_image_tokens"]
|
||||
cached_text_tokens: Final = max(details["cache_hit_tokens"] - cached_audio_tokens - cached_image_tokens, 0)
|
||||
configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost")
|
||||
cache_creation_cost: Final = (
|
||||
input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost
|
||||
)
|
||||
return (
|
||||
base_input_tokens * input_cost_per_token / 2
|
||||
+ cached_text_tokens * half_cache_read_rate
|
||||
+ cached_audio_tokens * cached_audio_rate
|
||||
+ cached_image_tokens * cached_image_rate
|
||||
+ cache_creation_tokens * cache_creation_cost / 2
|
||||
)
|
||||
|
||||
|
||||
def _batch_prompt_cost(
|
||||
|
|
@ -3340,6 +3375,7 @@ def _batch_prompt_cost(
|
|||
model_info: ModelInfo,
|
||||
input_rate: float,
|
||||
batch_cache_read_rate: float | None,
|
||||
batch_cache_read_image_rate: float | None,
|
||||
cache_creation_rate: float | None,
|
||||
cache_creation_rates_by_ttl: Mapping[str, float],
|
||||
) -> float:
|
||||
|
|
@ -3349,8 +3385,30 @@ def _batch_prompt_cost(
|
|||
if batch_cache_read_rate is None
|
||||
else batch_cache_read_rate
|
||||
)
|
||||
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
|
||||
cached_audio_rate: Final = _cached_modality_rate(
|
||||
model_info,
|
||||
"cache_read_input_audio_token_cost",
|
||||
_optional_batch_rate(model_info, "cache_read_input_audio_token_cost_batches", usage.prompt_tokens),
|
||||
batch_cache_read_rate,
|
||||
cache_read_rate,
|
||||
usage.prompt_tokens,
|
||||
)
|
||||
cached_image_rate: Final = _cached_modality_rate(
|
||||
model_info,
|
||||
"cache_read_input_image_token_cost",
|
||||
batch_cache_read_image_rate,
|
||||
batch_cache_read_rate,
|
||||
cache_read_rate,
|
||||
usage.prompt_tokens,
|
||||
)
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cached_audio_rate is not None else 0
|
||||
cached_image_tokens: Final = details["cached_image_tokens"] if cached_image_rate is not None else 0
|
||||
cached_text_tokens: Final = (
|
||||
max(details["cache_hit_tokens"] - details["cache_hit_audio_tokens"] - details["cached_image_tokens"], 0)
|
||||
if cache_read_rate is not None
|
||||
else 0
|
||||
)
|
||||
cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens
|
||||
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
|
||||
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
|
||||
audio_tokens, image_tokens, video_tokens = (
|
||||
|
|
@ -3367,13 +3425,9 @@ def _batch_prompt_cost(
|
|||
* _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate, prompt_tokens=usage.prompt_tokens)
|
||||
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
|
||||
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
|
||||
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
|
||||
+ cached_audio_tokens
|
||||
* _batch_rate(
|
||||
model_info,
|
||||
"cache_read_input_audio_token_cost_batches",
|
||||
_cached_audio_batch_fallback(model_info, batch_cache_read_rate, cache_read_rate, usage.prompt_tokens),
|
||||
)
|
||||
+ cached_text_tokens * (cache_read_rate or 0.0)
|
||||
+ cached_audio_tokens * (cached_audio_rate or 0.0)
|
||||
+ cached_image_tokens * (cached_image_rate or 0.0)
|
||||
+ (
|
||||
calculate_cache_writing_cost(
|
||||
written_tokens,
|
||||
|
|
|
|||
|
|
@ -112,6 +112,7 @@ _BATCH_RATE_PREFIXES: Final = (
|
|||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_read_input_image_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
)
|
||||
_BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values())
|
||||
|
|
@ -326,6 +327,7 @@ class BatchCostRates:
|
|||
cache_read: float | None
|
||||
cache_creation: float | None
|
||||
cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
|
||||
cache_read_image: float | None = None
|
||||
|
||||
|
||||
def _batch_rate(model_info: ModelInfo, key: str) -> float | None:
|
||||
|
|
@ -388,6 +390,7 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide
|
|||
if (rate := _batch_rate_for_prefix(model_info, key, usage, inclusive)) is not None
|
||||
}
|
||||
),
|
||||
cache_read_image=_batch_rate_for_prefix(model_info, "cache_read_input_image_token_cost", usage, inclusive),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -36449,6 +36449,7 @@
|
|||
},
|
||||
"gpt-image-1.5": {
|
||||
"cache_read_input_image_token_cost": 2e-06,
|
||||
"cache_read_input_image_token_cost_batches": 1e-06,
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost_batches": 6.3e-07,
|
||||
"deprecation_date": "2026-12-01",
|
||||
|
|
@ -36497,6 +36498,7 @@
|
|||
},
|
||||
"gpt-image-2": {
|
||||
"cache_read_input_image_token_cost": 2e-06,
|
||||
"cache_read_input_image_token_cost_batches": 1e-06,
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost_batches": 6.25e-07,
|
||||
"deprecation_date": "2027-02-10",
|
||||
|
|
@ -36539,6 +36541,7 @@
|
|||
},
|
||||
"gpt-image-2.5-flare": {
|
||||
"cache_read_input_image_token_cost": 2e-06,
|
||||
"cache_read_input_image_token_cost_batches": 1e-06,
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost_batches": 6.25e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
|
|
@ -36575,6 +36578,7 @@
|
|||
},
|
||||
"gpt-image-2.5-sunburst": {
|
||||
"cache_read_input_image_token_cost": 2e-06,
|
||||
"cache_read_input_image_token_cost_batches": 1e-06,
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost_batches": 6.25e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
|
|
@ -39936,6 +39940,7 @@
|
|||
},
|
||||
"gpt-image-1-mini": {
|
||||
"cache_read_input_image_token_cost": 2.5e-07,
|
||||
"cache_read_input_image_token_cost_batches": 1.3e-07,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_read_input_token_cost_batches": 1e-07,
|
||||
"deprecation_date": "2026-12-01",
|
||||
|
|
@ -60402,6 +60407,7 @@
|
|||
},
|
||||
"chatgpt-image-latest": {
|
||||
"cache_read_input_image_token_cost": 2e-06,
|
||||
"cache_read_input_image_token_cost_batches": 1e-06,
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost_batches": 6.3e-07,
|
||||
"deprecation_date": "2026-12-01",
|
||||
|
|
|
|||
|
|
@ -372,6 +372,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
cache_read_input_token_cost_above_512k_tokens: float | None
|
||||
cache_read_input_token_cost_batches: ReadOnly[float | None]
|
||||
cache_read_input_audio_token_cost_batches: ReadOnly[float | None]
|
||||
cache_read_input_image_token_cost_batches: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None]
|
||||
|
|
@ -4190,6 +4191,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
cache_read_input_token_cost_above_200k_tokens_batches: float | None = None
|
||||
cache_read_input_token_cost_above_272k_tokens_batches: float | None = None
|
||||
cache_read_input_audio_token_cost_batches: float | None = None
|
||||
cache_read_input_image_token_cost_batches: float | None = None
|
||||
cache_creation_input_token_cost_batches: float | None = None
|
||||
cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None
|
||||
cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None
|
||||
|
|
|
|||
|
|
@ -7266,6 +7266,9 @@ def get_model_info_helper(
|
|||
cache_read_input_audio_token_cost_batches=_model_info.get(
|
||||
"cache_read_input_audio_token_cost_batches", None
|
||||
),
|
||||
cache_read_input_image_token_cost_batches=_model_info.get(
|
||||
"cache_read_input_image_token_cost_batches", None
|
||||
),
|
||||
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
|
||||
cache_read_input_image_token_cost=_model_info.get("cache_read_input_image_token_cost", None),
|
||||
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
|
||||
|
|
|
|||
|
|
@ -36449,6 +36449,7 @@
|
|||
},
|
||||
"gpt-image-1.5": {
|
||||
"cache_read_input_image_token_cost": 2e-06,
|
||||
"cache_read_input_image_token_cost_batches": 1e-06,
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost_batches": 6.3e-07,
|
||||
"deprecation_date": "2026-12-01",
|
||||
|
|
@ -36497,6 +36498,7 @@
|
|||
},
|
||||
"gpt-image-2": {
|
||||
"cache_read_input_image_token_cost": 2e-06,
|
||||
"cache_read_input_image_token_cost_batches": 1e-06,
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost_batches": 6.25e-07,
|
||||
"deprecation_date": "2027-02-10",
|
||||
|
|
@ -36539,6 +36541,7 @@
|
|||
},
|
||||
"gpt-image-2.5-flare": {
|
||||
"cache_read_input_image_token_cost": 2e-06,
|
||||
"cache_read_input_image_token_cost_batches": 1e-06,
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost_batches": 6.25e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
|
|
@ -36575,6 +36578,7 @@
|
|||
},
|
||||
"gpt-image-2.5-sunburst": {
|
||||
"cache_read_input_image_token_cost": 2e-06,
|
||||
"cache_read_input_image_token_cost_batches": 1e-06,
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost_batches": 6.25e-07,
|
||||
"input_cost_per_token": 5e-06,
|
||||
|
|
@ -39936,6 +39940,7 @@
|
|||
},
|
||||
"gpt-image-1-mini": {
|
||||
"cache_read_input_image_token_cost": 2.5e-07,
|
||||
"cache_read_input_image_token_cost_batches": 1.3e-07,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"cache_read_input_token_cost_batches": 1e-07,
|
||||
"deprecation_date": "2026-12-01",
|
||||
|
|
@ -60402,6 +60407,7 @@
|
|||
},
|
||||
"chatgpt-image-latest": {
|
||||
"cache_read_input_image_token_cost": 2e-06,
|
||||
"cache_read_input_image_token_cost_batches": 1e-06,
|
||||
"cache_read_input_token_cost": 1.25e-06,
|
||||
"cache_read_input_token_cost_batches": 6.3e-07,
|
||||
"deprecation_date": "2026-12-01",
|
||||
|
|
|
|||
|
|
@ -4163,6 +4163,23 @@ def test_get_batch_cost_rates_parses_string_rates():
|
|||
assert (rates.input, rates.output, rates.cache_read) == (2e-6, 6e-6, 2e-7)
|
||||
|
||||
|
||||
def test_get_batch_cost_rates_reads_the_batch_cached_image_rate_for_the_crossed_tier():
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates
|
||||
|
||||
rates = get_batch_cost_rates(
|
||||
_batch_rates_model_info(
|
||||
input_cost_per_token_batches=2.5e-6,
|
||||
cache_read_input_image_token_cost=2e-6,
|
||||
cache_read_input_image_token_cost_batches=1e-6,
|
||||
cache_read_input_image_token_cost_above_272k_tokens_batches=1.5e-6,
|
||||
),
|
||||
Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001),
|
||||
"openai",
|
||||
)
|
||||
|
||||
assert rates.cache_read_image == 1.5e-6
|
||||
|
||||
|
||||
def test_get_batch_cost_rates_falls_back_to_the_flat_rates_when_tier_rates_are_unparsable():
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import asyncio
|
|||
import datetime
|
||||
import time
|
||||
from collections.abc import Iterator, Mapping
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import Final, Literal, cast
|
||||
|
||||
|
|
@ -10,7 +11,7 @@ import httpx
|
|||
import pytest
|
||||
import respx
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import TranscriptionResponse, model_cost
|
||||
|
|
@ -4580,6 +4581,158 @@ def test_batch_cost_calculator_bills_cached_tokens_at_the_standard_cache_price_w
|
|||
assert prompt_cost == pytest.approx(1000 * expected_rate)
|
||||
|
||||
|
||||
def _cached_image_batch_usage() -> Usage:
|
||||
return Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
total_tokens=1000,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cached_tokens=1000, image_tokens=1000, cached_tokens_details=CachedTokensDetails(image_tokens=1000)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("image_batch_cache_price", "text_batch_cache_price", "expected_rate"),
|
||||
[
|
||||
pytest.param(1e-6, 6.25e-7, 1e-6, id="the-batch-cached-image-price-wins"),
|
||||
pytest.param(None, 6.25e-7, 6.25e-7, id="without-it-cached-images-bill-the-batch-cache-price"),
|
||||
pytest.param(None, None, 2e-6, id="without-any-batch-cache-price-cached-images-bill-the-standard-image-cache-price"),
|
||||
],
|
||||
)
|
||||
def test_batch_cost_calculator_bills_cached_image_tokens_at_the_batch_cached_image_price(
|
||||
image_batch_cache_price: float | None, text_batch_cache_price: float | None, expected_rate: float
|
||||
) -> None:
|
||||
model_info: Final = cast(
|
||||
ModelInfo,
|
||||
{
|
||||
"input_cost_per_token": 5e-6,
|
||||
"input_cost_per_image_token": 8e-6,
|
||||
"cache_read_input_token_cost": 1.25e-6,
|
||||
"cache_read_input_image_token_cost": 2e-6,
|
||||
"input_cost_per_token_batches": 2.5e-6,
|
||||
"input_cost_per_image_token_batches": 4e-6,
|
||||
"cache_read_input_token_cost_batches": text_batch_cache_price,
|
||||
"cache_read_input_image_token_cost_batches": image_batch_cache_price,
|
||||
},
|
||||
)
|
||||
|
||||
prompt_cost, _ = batch_cost_calculator(
|
||||
usage=_cached_image_batch_usage(), model="gpt-image-2", custom_llm_provider="openai", model_info=model_info
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(1000 * expected_rate)
|
||||
|
||||
|
||||
_COST_MAP_ROWS: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
|
||||
|
||||
def _rows_with_a_batch_cached_image_price() -> tuple[tuple[str, str, float, ModelInfo], ...]:
|
||||
rows: Final = _COST_MAP_ROWS.validate_json(
|
||||
Path(litellm.__file__).with_name("model_prices_and_context_window_backup.json").read_bytes()
|
||||
)
|
||||
return tuple(
|
||||
(name, str(entry["litellm_provider"]), rate, cast(ModelInfo, entry))
|
||||
for name, entry in rows.items()
|
||||
if isinstance(rate := entry.get("cache_read_input_image_token_cost_batches"), float)
|
||||
)
|
||||
|
||||
|
||||
def _batch_cached_image_cost(name: str, provider: str, entry: ModelInfo) -> float:
|
||||
return batch_cost_calculator(_cached_image_batch_usage(), name, provider, model_info=entry)[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("usage", "modality_batch_cache_key"),
|
||||
[
|
||||
pytest.param(_cached_audio_batch_usage(), "cache_read_input_audio_token_cost_batches", id="cached-audio"),
|
||||
pytest.param(_cached_image_batch_usage(), "cache_read_input_image_token_cost_batches", id="cached-image"),
|
||||
],
|
||||
)
|
||||
def test_batch_cost_calculator_bills_cached_modality_tokens_at_their_batch_cache_price_without_a_text_cache_price(
|
||||
usage: Usage, modality_batch_cache_key: str
|
||||
) -> None:
|
||||
model_info: Final = cast(
|
||||
ModelInfo,
|
||||
{
|
||||
"input_cost_per_token": 5e-6,
|
||||
"input_cost_per_audio_token": 8e-6,
|
||||
"input_cost_per_image_token": 8e-6,
|
||||
"input_cost_per_token_batches": 2.5e-6,
|
||||
"input_cost_per_audio_token_batches": 4e-6,
|
||||
"input_cost_per_image_token_batches": 4e-6,
|
||||
modality_batch_cache_key: 1e-6,
|
||||
},
|
||||
)
|
||||
|
||||
prompt_cost, _ = batch_cost_calculator(
|
||||
usage=usage, model="gpt-image-2", custom_llm_provider="openai", model_info=model_info
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(1000 * 1e-6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("usage", "modality_cache_key", "expected_prompt_cost"),
|
||||
[
|
||||
pytest.param(
|
||||
_cached_audio_batch_usage(),
|
||||
"cache_read_input_audio_token_cost_batches",
|
||||
1000 * 1e-6,
|
||||
id="cached-audio-batch-price",
|
||||
),
|
||||
pytest.param(
|
||||
_cached_image_batch_usage(),
|
||||
"cache_read_input_image_token_cost_batches",
|
||||
1000 * 1e-6,
|
||||
id="cached-image-batch-price",
|
||||
),
|
||||
pytest.param(
|
||||
_cached_audio_batch_usage(),
|
||||
"cache_read_input_audio_token_cost",
|
||||
1000 * 1e-6 / 2,
|
||||
id="cached-audio-standard-price-halved",
|
||||
),
|
||||
pytest.param(
|
||||
_cached_image_batch_usage(),
|
||||
"cache_read_input_image_token_cost",
|
||||
1000 * 1e-6 / 2,
|
||||
id="cached-image-standard-price-halved",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_batch_cost_calculator_half_price_fallback_bills_cached_modality_tokens_at_their_own_cache_price(
|
||||
usage: Usage, modality_cache_key: str, expected_prompt_cost: float
|
||||
) -> None:
|
||||
model_info: Final = cast(
|
||||
ModelInfo,
|
||||
{
|
||||
"input_cost_per_token": 5e-6,
|
||||
"cache_read_input_token_cost": 2.5e-6,
|
||||
modality_cache_key: 1e-6,
|
||||
},
|
||||
)
|
||||
|
||||
prompt_cost, _ = batch_cost_calculator(
|
||||
usage=usage, model="gpt-image-2", custom_llm_provider="openai", model_info=model_info
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(expected_prompt_cost)
|
||||
|
||||
|
||||
def test_batch_cached_image_tokens_bill_the_rows_batch_cached_image_price_on_every_row_that_has_one() -> None:
|
||||
rows: Final = _rows_with_a_batch_cached_image_price()
|
||||
assert rows, "no cost map row prices Batch cached image input"
|
||||
|
||||
mispriced: Final = tuple(
|
||||
(name, _batch_cached_image_cost(name, provider, entry), 1000 * rate)
|
||||
for name, provider, rate, entry in rows
|
||||
if _batch_cached_image_cost(name, provider, entry) != pytest.approx(1000 * rate)
|
||||
)
|
||||
|
||||
assert mispriced == ()
|
||||
|
||||
|
||||
def test_batch_cost_calculator_bills_long_context_cached_tokens_at_the_standard_long_context_cache_price() -> None:
|
||||
model_info: Final[ModelInfo] = {
|
||||
"input_cost_per_token": 1.25e-6,
|
||||
|
|
|
|||
|
|
@ -1023,6 +1023,7 @@ _COST_MAP_ROW_SCHEMA: Final = {
|
|||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"},
|
||||
"cache_read_input_audio_token_cost": {"type": "number"},
|
||||
"cache_read_input_audio_token_cost_batches": {"type": "number"},
|
||||
"cache_read_input_image_token_cost_batches": {"type": "number"},
|
||||
"cache_read_input_audio_token_cost_flex": {"type": "number"},
|
||||
"cache_read_input_audio_token_cost_priority": {"type": "number"},
|
||||
"cache_read_input_image_token_cost": {"type": "number"},
|
||||
|
|
|
|||
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
4
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -36614,6 +36614,8 @@ export interface components {
|
|||
cache_read_input_audio_token_cost_priority?: number | null;
|
||||
/** Cache Read Input Image Token Cost */
|
||||
cache_read_input_image_token_cost?: number | null;
|
||||
/** Cache Read Input Image Token Cost Batches */
|
||||
cache_read_input_image_token_cost_batches?: number | null;
|
||||
/** Cache Read Input Token Cost */
|
||||
cache_read_input_token_cost?: number | null;
|
||||
/** Cache Read Input Token Cost Above 100K Tokens */
|
||||
|
|
@ -52804,6 +52806,8 @@ export interface components {
|
|||
cache_read_input_audio_token_cost_priority?: number | null;
|
||||
/** Cache Read Input Image Token Cost */
|
||||
cache_read_input_image_token_cost?: number | null;
|
||||
/** Cache Read Input Image Token Cost Batches */
|
||||
cache_read_input_image_token_cost_batches?: number | null;
|
||||
/** Cache Read Input Token Cost */
|
||||
cache_read_input_token_cost?: number | null;
|
||||
/** Cache Read Input Token Cost Above 100K Tokens */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue