scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-28 06:23:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 94

fix(cost): bill gemini audio input and cached audio at flex, priority and batch rates (#48086)

Browse source 
* fix(gemini): price tiered audio token rates

* fix: correct Gemini audio pricing catalog

* fix(gemini): preserve cached modality pricing

---------

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-10-24 09:12:31 -07:00 • committed by GitHub
parent 455923014a
commit dc0237ccd7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 429 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
litellm/cost_calculator.py
View file

@ -2879,6 +2879,7 @@ def _batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"cache_read_input_audio_token_cost_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
@ -3012,6 +3013,7 @@ def _batch_prompt_cost(
) -> float:
details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
audio_tokens, image_tokens, video_tokens = (
details["audio_tokens"],
@ -3026,7 +3028,9 @@ def _batch_prompt_cost(
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ cached_tokens * (cache_read_rate or 0.0)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
+ cached_audio_tokens
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
+ written_tokens * (cache_creation_rate or 0.0)
)

29
litellm/litellm_core_utils/llm_cost_calc/zero_cost_diagnostic.py
View file

@ -5,6 +5,7 @@ from typing import Final
from pydantic import TypeAdapter, ValidationError
from typing_extensions import assert_never
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
@ -16,6 +17,8 @@ ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total"
_TEXT_INPUT_RATE: Final = "input_cost_per_token"
_AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token"
_CACHE_READ_INPUT_RATE: Final = "cache_read_input_token_cost"
_CACHE_READ_AUDIO_INPUT_RATE: Final = "cache_read_input_audio_token_cost"
_TEXT_OUTPUT_RATE: Final = "output_cost_per_token"
_AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token"
_RATE_KEY_MARKERS: Final = ("cost", "pricing")
@ -33,19 +36,38 @@ def _tokens(value: object) -> int:
def used_pricing_keys(usage: Usage) -> tuple[str, ...]:
prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details)
prompt_details: Final = parse_prompt_tokens_details(usage)
prompt_audio: Final = prompt_details["audio_tokens"]
completion_audio: Final = _audio_tokens(usage.completion_tokens_details)
prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio
prompt_text: Final = max(
_tokens(usage.prompt_tokens)
- prompt_details["cache_hit_tokens"]
- prompt_audio
- prompt_details["image_tokens"]
- prompt_details["video_tokens"],
0,
)
cached_text: Final = max(prompt_details["cache_hit_tokens"] - prompt_details["cache_hit_audio_tokens"], 0)
completion_text: Final = _tokens(usage.completion_tokens) - completion_audio
components: Final = (
(_TEXT_INPUT_RATE, prompt_text),
(_AUDIO_INPUT_RATE, prompt_audio),
(_CACHE_READ_INPUT_RATE, cached_text),
(_CACHE_READ_AUDIO_INPUT_RATE, prompt_details["cache_hit_audio_tokens"]),
(_TEXT_OUTPUT_RATE, completion_text),
(_AUDIO_OUTPUT_RATE, completion_audio),
)
return tuple(key for key, count in components if count > 0)
def _effective_pricing_key(key: str, pricing_entry: Mapping[str, object]) -> str:
if key == _CACHE_READ_AUDIO_INPUT_RATE and pricing_entry.get(key) is None:
return _CACHE_READ_INPUT_RATE if pricing_entry.get(_CACHE_READ_INPUT_RATE) is not None else _TEXT_INPUT_RATE
if key == _CACHE_READ_INPUT_RATE and pricing_entry.get(key) is None:
return _TEXT_INPUT_RATE
return key
def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None:
try:
return _NESTED_PRICING.validate_python(value)
@ -89,7 +111,8 @@ def diagnose_zero_cost(
pricing_entry: Mapping[str, object],
calculation_failed: bool,
) -> StandardLoggingZeroCostDiagnostic | None:
used_keys: Final = used_pricing_keys(usage)
effective_keys: Final = tuple(_effective_pricing_key(key, pricing_entry) for key in used_pricing_keys(usage))
used_keys: Final = tuple(key for index, key in enumerate(effective_keys) if key not in effective_keys[:index])
if not used_keys:
return None
missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None)

31
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -42,6 +42,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_httpx_client,
)
from litellm.types.llms.anthropic import AnthropicThinkingParam
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
from litellm.types.llms.openai import (
AllMessageValues,
@ -2364,26 +2365,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
elif modality == "DOCUMENT":
cached_text_tokens = (cached_text_tokens or 0) + token_count
## Calculate non-cached tokens by subtracting cached from total (per modality)
## This is necessary because promptTokensDetails includes both cached and non-cached tokens
## See: https://github.com/BerriAI/litellm/issues/18750
if cached_text_tokens is not None and prompt_text_tokens is not None:
# Explicit caching: subtract cached tokens per modality from cacheTokensDetails
prompt_text_tokens = prompt_text_tokens - cached_text_tokens
elif (
if (
cached_tokens is not None
and prompt_text_tokens is not None
and cached_text_tokens is None
and "cacheTokensDetails" not in usage_metadata
):
# Implicit caching: only cachedContentTokenCount is provided (no cacheTokensDetails)
# Subtract from text tokens since implicit caching is primarily for text content
# See: https://github.com/BerriAI/litellm/issues/16341
prompt_text_tokens = prompt_text_tokens - cached_tokens
if cached_audio_tokens is not None and prompt_audio_tokens is not None:
prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens
if cached_image_tokens is not None and prompt_image_tokens is not None:
prompt_image_tokens = prompt_image_tokens - cached_image_tokens
if cached_video_tokens is not None and prompt_video_tokens is not None:
prompt_video_tokens = prompt_video_tokens - cached_video_tokens
@ -2404,9 +2392,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
billable_text_tokens: Final = (
prompt_text_tokens if prompt_text_tokens is None else prompt_text_tokens + billable_tool_use_prompt_tokens
)
cached_tokens_details_fields: Final = MappingProxyType(
{
key: token_count
for key, token_count in (
("text_tokens", cached_text_tokens),
("audio_tokens", cached_audio_tokens),
("image_tokens", cached_image_tokens),
)
if token_count is not None
}
)
cached_tokens_details: Final = (
CachedTokensDetails.model_validate(cached_tokens_details_fields) if cached_tokens_details_fields else None
)
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cached_tokens,
cached_tokens_details=cached_tokens_details,
audio_tokens=prompt_audio_tokens,
text_tokens=billable_text_tokens,
image_tokens=prompt_image_tokens,

16
litellm/model_prices_and_context_window_backup.json
View file

@ -31137,11 +31137,15 @@
},
"gemini/gemini-2.5-flash": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 3e-08,
"cache_read_input_token_cost_batches": 3e-08,
"cache_read_input_token_cost_flex": 3e-08,
"cache_read_input_token_cost_priority": 5.4e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 3e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31466,11 +31470,15 @@
},
"gemini/gemini-2.5-flash-lite": {
"cache_read_input_audio_token_cost": 3e-08,
"cache_read_input_audio_token_cost_batches": 3e-08,
"cache_read_input_audio_token_cost_flex": 3e-08,
"cache_read_input_audio_token_cost_priority": 5.4e-08,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_batches": 1e-08,
"cache_read_input_token_cost_flex": 1e-08,
"cache_read_input_token_cost_priority": 1.8e-08,
"input_cost_per_audio_token": 3e-07,
"input_cost_per_audio_token_flex": 1.5e-07,
"input_cost_per_token": 1e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31760,12 +31768,16 @@
},
"gemini/gemini-3.1-flash-lite": {
"cache_read_input_audio_token_cost": 5e-08,
"cache_read_input_audio_token_cost_batches": 2.5e-08,
"cache_read_input_audio_token_cost_flex": 2.5e-08,
"cache_read_input_audio_token_cost_priority": 9e-08,
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"deprecation_date": "2027-05-07",
"input_cost_per_audio_token": 5e-07,
"input_cost_per_audio_token_flex": 2.5e-07,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"input_cost_per_token_flex": 1.25e-07,
@ -31886,10 +31898,14 @@
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 5e-08,
"cache_read_input_token_cost_batches": 5e-08,
"cache_read_input_token_cost_flex": 5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,

10
litellm/types/utils.py
View file

@ -333,6 +333,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_read_input_token_cost: float | None
cache_read_input_audio_token_cost: ReadOnly[float | None]
cache_read_input_audio_token_cost_flex: ReadOnly[float | None]
cache_read_input_audio_token_cost_priority: ReadOnly[float | None]
cache_read_input_image_token_cost: ReadOnly[float | None]
explicit_cache_read_input_token_cost: ReadOnly[float | None]
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
@ -348,6 +350,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_read_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
cache_read_input_token_cost_above_512k_tokens: float | None
cache_read_input_token_cost_batches: ReadOnly[float | None]
cache_read_input_audio_token_cost_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None]
@ -361,6 +364,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates
input_cost_per_character: float | None # only for vertex ai models
input_cost_per_audio_token: float | None
input_cost_per_audio_token_flex: ReadOnly[float | None]
input_cost_per_audio_token_priority: ReadOnly[float | None]
input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models
input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models
input_cost_per_token_above_100k_tokens: ReadOnly[float | None]
@ -4020,14 +4025,19 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
cache_read_input_token_cost_above_100k_tokens_batches: float | None = None
cache_read_input_token_cost_above_200k_tokens_batches: float | None = None
cache_read_input_token_cost_above_272k_tokens_batches: float | None = None
cache_read_input_audio_token_cost_batches: float | None = None
cache_creation_input_token_cost_batches: float | None = None
cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None
cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None
cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None
cache_read_input_audio_token_cost: float | None = None
cache_read_input_audio_token_cost_flex: float | None = None
cache_read_input_audio_token_cost_priority: float | None = None
cache_read_input_image_token_cost: float | None = None
input_cost_per_character_above_128k_tokens: float | None = None
input_cost_per_audio_token: float | None = None
input_cost_per_audio_token_flex: float | None = None
input_cost_per_audio_token_priority: float | None = None
input_cost_per_token_cache_hit: float | None = None
input_cost_per_token_above_128k_tokens: float | None = None
input_cost_per_token_above_100k_tokens: float | None = None

9
litellm/utils.py
View file

@ -6773,6 +6773,13 @@ def get_model_info_helper(
),
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
cache_read_input_audio_token_cost_flex=_model_info.get("cache_read_input_audio_token_cost_flex", None),
cache_read_input_audio_token_cost_priority=_model_info.get(
"cache_read_input_audio_token_cost_priority", None
),
cache_read_input_audio_token_cost_batches=_model_info.get(
"cache_read_input_audio_token_cost_batches", None
),
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
@ -6858,6 +6865,8 @@ def get_model_info_helper(
input_cost_per_second=_model_info.get("input_cost_per_second", None),
input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None),
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None),
input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None),
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None),

16
model_prices_and_context_window.json
View file

@ -31137,11 +31137,15 @@
},
"gemini/gemini-2.5-flash": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 3e-08,
"cache_read_input_token_cost_batches": 3e-08,
"cache_read_input_token_cost_flex": 3e-08,
"cache_read_input_token_cost_priority": 5.4e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 3e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31466,11 +31470,15 @@
},
"gemini/gemini-2.5-flash-lite": {
"cache_read_input_audio_token_cost": 3e-08,
"cache_read_input_audio_token_cost_batches": 3e-08,
"cache_read_input_audio_token_cost_flex": 3e-08,
"cache_read_input_audio_token_cost_priority": 5.4e-08,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_batches": 1e-08,
"cache_read_input_token_cost_flex": 1e-08,
"cache_read_input_token_cost_priority": 1.8e-08,
"input_cost_per_audio_token": 3e-07,
"input_cost_per_audio_token_flex": 1.5e-07,
"input_cost_per_token": 1e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31760,12 +31768,16 @@
},
"gemini/gemini-3.1-flash-lite": {
"cache_read_input_audio_token_cost": 5e-08,
"cache_read_input_audio_token_cost_batches": 2.5e-08,
"cache_read_input_audio_token_cost_flex": 2.5e-08,
"cache_read_input_audio_token_cost_priority": 9e-08,
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"deprecation_date": "2027-05-07",
"input_cost_per_audio_token": 5e-07,
"input_cost_per_audio_token_flex": 2.5e-07,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"input_cost_per_token_flex": 1.25e-07,
@ -31886,10 +31898,14 @@
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 5e-08,
"cache_read_input_token_cost_batches": 5e-08,
"cache_read_input_token_cost_flex": 5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,

62
tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py
View file

@ -11,6 +11,7 @@ import respx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
TIER_MODEL: Final = "tier-priced-test-model"
@ -31,6 +32,20 @@ PROMPT_TOKENS: Final = 1000
CACHED_TOKENS: Final = 200
COMPLETION_TOKENS: Final = 500
TIER_API_BASE: Final = "https://tier-pricing.invalid/v1"
AUDIO_TIER_MODEL: Final = "tier-priced-gemini-audio-test-model"
AUDIO_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
{
"input_cost_per_token": 4e-06,
"output_cost_per_token": 8e-06,
"cache_read_input_token_cost": 1e-06,
"input_cost_per_audio_token": 3e-06,
"cache_read_input_audio_token_cost": 2e-06,
"input_cost_per_audio_token_flex": 1e-06,
"input_cost_per_audio_token_priority": 2e-06,
"cache_read_input_audio_token_cost_flex": 4e-07,
"cache_read_input_audio_token_cost_priority": 8e-07,
}
)
def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float:
@ -78,6 +93,53 @@ def test_completion_cost_bills_the_price_columns_of_the_service_tier(
assert cost == pytest.approx(_cost_at(TIER_ROW, column_suffix))
@pytest.mark.parametrize(
("service_tier", "audio_rate_key", "cache_audio_rate_key"),
(
pytest.param("flex", "input_cost_per_audio_token_flex", "cache_read_input_audio_token_cost_flex"),
pytest.param("priority", "input_cost_per_audio_token_priority", "cache_read_input_audio_token_cost_priority"),
),
)
def test_gemini_audio_and_cached_audio_use_service_tier_rates(
local_model_cost_map: None, service_tier: str, audio_rate_key: str, cache_audio_rate_key: str
) -> None:
litellm.register_model(
{
AUDIO_TIER_MODEL: {
"litellm_provider": "gemini",
"mode": "chat",
**AUDIO_TIER_ROW,
}
}
)
usage: Final = Usage(
prompt_tokens=40,
completion_tokens=0,
total_tokens=40,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=20,
audio_tokens=20,
cached_tokens=10,
cached_tokens_details=CachedTokensDetails(audio_tokens=10),
),
)
response: Final = ModelResponse(model=AUDIO_TIER_MODEL, usage=usage)
cost: Final = litellm.completion_cost(
completion_response=response,
model=AUDIO_TIER_MODEL,
custom_llm_provider="gemini",
service_tier=service_tier,
)
expected: Final = (
20 * AUDIO_TIER_ROW["input_cost_per_token"]
+ 10 * AUDIO_TIER_ROW[audio_rate_key]
+ 10 * AUDIO_TIER_ROW[cache_audio_rate_key]
)
assert cost == pytest.approx(expected)
LONG_CONTEXT_TIER_MODEL: Final = "long-context-tier-priced-test-model"
LONG_CONTEXT_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
{

31
tests/unit/litellm_core_utils/llm_cost_calc/test_zero_cost_diagnostic.py
View file

@ -9,6 +9,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import (
used_pricing_keys,
zero_cost_warning,
)
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.utils import CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, Usage
PER_SECOND_ENTRY: Final = {"cost_per_second": 0.00042}
@ -135,6 +136,36 @@ def test_audio_tokens_need_the_audio_rates() -> None:
assert diagnostic["missing_pricing_keys"] == ("input_cost_per_audio_token", "output_cost_per_audio_token")
def test_cached_audio_uses_its_cache_rate_in_zero_cost_diagnostic() -> None:
usage: Final = Usage(
prompt_tokens=20,
completion_tokens=0,
total_tokens=20,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=0,
audio_tokens=20,
cached_tokens=20,
cached_tokens_details=CachedTokensDetails(audio_tokens=20),
),
)
pricing_entry: Final = {
"input_cost_per_token": 1e-6,
"input_cost_per_audio_token": 2e-6,
"cache_read_input_audio_token_cost": 0,
}
assert used_pricing_keys(usage) == ("cache_read_input_audio_token_cost",)
assert (
diagnose_zero_cost(
usage=usage,
pricing_model="gemini-audio",
pricing_entry=pricing_entry,
calculation_failed=False,
)
is None
)
def test_warning_names_the_request_the_entry_the_missing_keys_and_the_counter() -> None:
diagnostic = diagnose_zero_cost(
usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False

147
tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
View file

@ -16,8 +16,10 @@ from pydantic import BaseModel
import litellm
from litellm import ModelResponse, completion, embedding, image_generation
from litellm._logging import verbose_logger
from litellm.cost_calculator import batch_cost_calculator
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details
from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
@ -30,8 +32,9 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
normalize_gemini_speech_config,
reject_unknown_gemini_tts_voice_keys,
)
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata
from litellm.types.utils import ChoiceLogprobs, Usage
from litellm.types.utils import ChoiceLogprobs, ModelInfo, Usage
from litellm.utils import _invalidate_model_cost_lowercase_map, CustomStreamWrapper
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.integrations.custom_logger import CustomLogger
@ -1515,6 +1518,7 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities():
v = VertexGeminiConfig()
usage_metadata = {
"promptTokenCount": 210,
"cachedContentTokenCount": 54,
"candidatesTokenCount": 50,
"totalTokenCount": 260,
"promptTokensDetails": [
@ -1536,9 +1540,10 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities():
usage_metadata = UsageMetadata(**usage_metadata)
result = v.calculate_usage(completion_response={"usageMetadata": usage_metadata})
# prompt details are total - cached per modality
assert result.prompt_tokens_details.text_tokens == 16 # 20 - 4
assert result.prompt_tokens_details.image_tokens == 140 # (90 + 100) - (40 + 10)
assert result.prompt_tokens_details.text_tokens == 20
assert result.prompt_tokens_details.image_tokens == 190
assert result.prompt_tokens_details.cached_tokens == 54
assert result.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=4, image_tokens=50)
# candidates details accumulate duplicate modalities
assert result.completion_tokens_details.text_tokens == 20 # 15 + 5
@ -1546,6 +1551,118 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities():
assert result.completion_tokens == 50
def _gemini_usage_with_cached_audio(cached_audio_tokens: int) -> Usage:
cache_tokens_details: Final = (
(
{"modality": "TEXT", "tokenCount": 5},
{"modality": "AUDIO", "tokenCount": cached_audio_tokens},
)
if cached_audio_tokens > 0
else ({"modality": "TEXT", "tokenCount": 5},)
)
completion_response: Final = {
"usageMetadata": {
"promptTokenCount": 40,
"cachedContentTokenCount": 5 + cached_audio_tokens,
"promptTokensDetails": (
{"modality": "TEXT", "tokenCount": 20},
{"modality": "AUDIO", "tokenCount": 20},
),
"cacheTokensDetails": cache_tokens_details,
"candidatesTokenCount": 7,
"totalTokenCount": 47,
}
}
return VertexGeminiConfig.calculate_usage(completion_response=completion_response)
def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_batch_rate() -> None:
model_info: Final[ModelInfo] = {
"input_cost_per_token": 2e-6,
"input_cost_per_token_batches": 1e-6,
"input_cost_per_audio_token": 6e-6,
"input_cost_per_audio_token_batches": 3e-6,
"cache_read_input_token_cost": 8e-7,
"cache_read_input_token_cost_batches": 4e-7,
"cache_read_input_audio_token_cost": 5e-7,
"cache_read_input_audio_token_cost_batches": 2e-7,
}
usage: Final = _gemini_usage_with_cached_audio(5)
expected_prompt_cost: Final = (
15 * model_info["input_cost_per_token_batches"]
+ 15 * model_info["input_cost_per_audio_token_batches"]
+ 5 * model_info["cache_read_input_token_cost_batches"]
+ 5 * model_info["cache_read_input_audio_token_cost_batches"]
)
prompt_cost, _ = batch_cost_calculator(
usage=usage,
model="gemini-audio-batch-test-model",
custom_llm_provider="gemini",
model_info=model_info,
)
assert prompt_cost == pytest.approx(expected_prompt_cost)
def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_standard_rate() -> None:
model_info: Final[ModelInfo] = {
"input_cost_per_token": 2e-6,
"input_cost_per_audio_token": 6e-6,
"cache_read_input_token_cost": 8e-7,
"cache_read_input_audio_token_cost": 5e-7,
}
usage: Final = _gemini_usage_with_cached_audio(5)
expected_prompt_cost: Final = (
15 * model_info["input_cost_per_token"]
+ 15 * model_info["input_cost_per_audio_token"]
+ 5 * model_info["cache_read_input_token_cost"]
+ 5 * model_info["cache_read_input_audio_token_cost"]
)
prompt_cost, _ = generic_cost_per_token(
model="gemini-audio-standard-test-model",
usage=usage,
custom_llm_provider="gemini",
model_info=model_info,
)
assert prompt_cost == pytest.approx(expected_prompt_cost)
def test_gemini_usage_totals_are_subtracted_once_and_audio_cache_rate_matches_audio_rate() -> None:
model_info: Final[ModelInfo] = {
"input_cost_per_token": 2e-6,
"input_cost_per_audio_token": 6e-6,
"cache_read_input_token_cost": 8e-7,
"cache_read_input_audio_token_cost": 6e-6,
}
cached_usage: Final = _gemini_usage_with_cached_audio(5)
uncached_usage: Final = _gemini_usage_with_cached_audio(0)
cached_details: Final = parse_prompt_tokens_details(cached_usage)
cached_prompt_cost, _ = generic_cost_per_token(
model="gemini-audio-standard-test-model",
usage=cached_usage,
custom_llm_provider="gemini",
model_info=model_info,
)
uncached_prompt_cost, _ = generic_cost_per_token(
model="gemini-audio-standard-test-model",
usage=uncached_usage,
custom_llm_provider="gemini",
model_info=model_info,
)
assert cached_usage.prompt_tokens_details.text_tokens == 20
assert cached_usage.prompt_tokens_details.audio_tokens == 20
assert cached_usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(
text_tokens=5, audio_tokens=5
)
assert cached_details["text_tokens"] == 15
assert cached_details["audio_tokens"] == 15
assert cached_prompt_cost == pytest.approx(uncached_prompt_cost)
def test_vertex_ai_map_thinking_param_with_budget_tokens_0():
"""
If budget_tokens is 0, do not set includeThoughts to True
@ -6150,7 +6267,7 @@ def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt():
def test_vertex_ai_usage_metadata_with_document_tokens_cached():
"""Test that cached DOCUMENT tokens are correctly subtracted from prompt text tokens."""
"""Test cached DOCUMENT token details and unchanged prompt cost."""
v = VertexGeminiConfig()
usage_metadata_dict = {
@ -6174,10 +6291,24 @@ def test_vertex_ai_usage_metadata_with_document_tokens_cached():
completion_response = {"usageMetadata": usage_metadata_dict}
result = v.calculate_usage(completion_response=completion_response)
# DOCUMENT cached tokens map to cached_text_tokens, so:
# text_tokens = (8 TEXT + 774 DOCUMENT) - 400 cached = 382
assert result.prompt_tokens_details.text_tokens == 382, "text_tokens should be (8 + 774) - 400 cached = 382"
assert result.prompt_tokens_details.text_tokens == 782
assert result.prompt_tokens_details.cached_tokens == 400
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 400
model_info: Final[ModelInfo] = {
"input_cost_per_token": 2e-6,
"cache_read_input_token_cost": 5e-7,
}
prompt_cost, _ = generic_cost_per_token(
model="gemini-document-cache-test-model",
usage=result,
custom_llm_provider="gemini",
model_info=model_info,
)
assert prompt_cost == pytest.approx(
382 * model_info["input_cost_per_token"] + 400 * model_info["cache_read_input_token_cost"]
)
def test_async_streaming_uses_custom_client():

77
tests/unit/test_cost_calculator.py
View file

@ -16,6 +16,7 @@ import litellm
from litellm import TranscriptionResponse, model_cost
from litellm.constants import REALTIME_METERED_USAGE_KEY
from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
@ -3337,14 +3338,7 @@ def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(_loca
def test_gemini_cache_tokens_details_no_negative_values():
"""
Test for Issue #18750: Negative text_tokens with Gemini caching
When using Gemini with explicit caching, the response includes cacheTokensDetails
which breaks down cached tokens by modality. This test ensures that:
1. text_tokens is never negative
2. We correctly subtract cached tokens per modality (not total)
"""
"""Ensure issue #18750 cache details remain non-negative after cost parsing."""
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
@ -3372,27 +3366,17 @@ def test_gemini_cache_tokens_details_no_negative_values():
usage = VertexGeminiConfig.calculate_usage(completion_response)
# Text tokens should be non-cached text only: 9402 - 9393 = 9
assert usage.prompt_tokens_details.text_tokens == 9, (
f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}"
)
prompt_details: Final = parse_prompt_tokens_details(usage)
# Image tokens should be non-cached image only: 258 - 258 = 0
assert usage.prompt_tokens_details.image_tokens == 0, (
f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}"
assert usage.prompt_tokens_details.text_tokens == 9402
assert usage.prompt_tokens_details.image_tokens == 258
assert usage.prompt_tokens_details.cached_tokens == 9651
assert usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(
text_tokens=9393, image_tokens=258
)
# Total cached should match
assert usage.prompt_tokens_details.cached_tokens == 9651, (
f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}"
)
# MOST IMPORTANT: text_tokens should NEVER be negative
assert usage.prompt_tokens_details.text_tokens >= 0, (
f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750"
)
print("✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative")
assert prompt_details["text_tokens"] == 9
assert prompt_details["image_tokens"] == 0
assert prompt_details["text_tokens"] >= 0
def test_gemini_without_cache_tokens_details():
@ -5990,6 +5974,45 @@ def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_t
assert prompt_cost == pytest.approx(298_926 * 3e-6 + 64 * 5e-6 + 10 * 3e-6 + 1_000 * 3e-7)
def test_batch_cost_calculator_prices_cached_audio_at_its_batch_rate() -> None:
model_info: Final[ModelInfo] = {
"input_cost_per_token": 2e-6,
"input_cost_per_token_batches": 1e-6,
"input_cost_per_audio_token": 6e-6,
"input_cost_per_audio_token_batches": 3e-6,
"cache_read_input_token_cost": 8e-7,
"cache_read_input_token_cost_batches": 4e-7,
"cache_read_input_audio_token_cost": 5e-7,
"cache_read_input_audio_token_cost_batches": 2e-7,
}
usage: Final = Usage(
prompt_tokens=40,
completion_tokens=0,
total_tokens=40,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=20,
audio_tokens=20,
cached_tokens=10,
cached_tokens_details=CachedTokensDetails(text_tokens=5, audio_tokens=5),
),
)
prompt_cost, _ = batch_cost_calculator(
usage=usage,
model="gemini-audio-batch-test-model",
custom_llm_provider="gemini",
model_info=model_info,
)
expected: Final = (
15 * model_info["input_cost_per_token_batches"]
+ 15 * model_info["input_cost_per_audio_token_batches"]
+ 5 * model_info["cache_read_input_token_cost_batches"]
+ 5 * model_info["cache_read_input_audio_token_cost_batches"]
)
assert prompt_cost == pytest.approx(expected)
QWEN3_NEXT_REGIONS: Final = ("ap-northeast-1", "ap-south-1", "ap-southeast-2", "eu-west-1", "eu-west-2", "sa-east-1")

4
tests/unit/test_utils.py
View file

@ -994,6 +994,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"},
"cache_read_input_audio_token_cost": {"type": "number"},
"cache_read_input_audio_token_cost_batches": {"type": "number"},
"cache_read_input_audio_token_cost_flex": {"type": "number"},
"cache_read_input_audio_token_cost_priority": {"type": "number"},
"cache_read_input_image_token_cost": {"type": "number"},
"audio_transcription_config": {"type": "string"},
"deprecation_date": {"type": "string"},
@ -1001,6 +1004,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"},
"google_maps_grounding_cost_per_query": {"type": "number"},
"input_cost_per_audio_token": {"type": "number"},
"input_cost_per_audio_token_flex": {"type": "number"},
"input_cost_per_image_token": {"type": "number"},
"input_cost_per_character": {"type": "number"},
"input_cost_per_character_above_128k_tokens": {"type": "number"},

24
tests/unit/test_utils_get_model_info.py
View file

@ -61,6 +61,30 @@ def test_get_model_info_custom_llm_with_same_name_vllm(monkeypatch):
assert model_info["input_cost_per_token"] == 0.0
def test_get_model_info_passes_through_gemini_tiered_audio_rates() -> None:
model: Final = "gemini-tiered-audio-rates-test"
prices: Final = {
"input_cost_per_audio_token_flex": 1e-6,
"input_cost_per_audio_token_priority": 2e-6,
"cache_read_input_audio_token_cost_flex": 3e-7,
"cache_read_input_audio_token_cost_priority": 4e-7,
"cache_read_input_audio_token_cost_batches": 5e-7,
}
litellm.register_model(
{
model: {
"litellm_provider": "gemini",
"mode": "chat",
**prices,
}
}
)
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="gemini")
assert {key: model_info[key] for key in prices} == prices
def test_get_model_info_ollama_chat():
from litellm.llms.ollama.completion.transformation import OllamaConfig

20
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -35259,6 +35259,12 @@ export interface components {
cache_creation_input_token_cost_ultrafast?: number | null;
/** Cache Read Input Audio Token Cost */
cache_read_input_audio_token_cost?: number | null;
/** Cache Read Input Audio Token Cost Batches */
cache_read_input_audio_token_cost_batches?: number | null;
/** Cache Read Input Audio Token Cost Flex */
cache_read_input_audio_token_cost_flex?: number | null;
/** Cache Read Input Audio Token Cost Priority */
cache_read_input_audio_token_cost_priority?: number | null;
/** Cache Read Input Image Token Cost */
cache_read_input_image_token_cost?: number | null;
/** Cache Read Input Token Cost */
@ -35338,6 +35344,10 @@ export interface components {
input_cost_per_audio_token?: number | null;
/** Input Cost Per Audio Token Batches */
input_cost_per_audio_token_batches?: number | null;
/** Input Cost Per Audio Token Flex */
input_cost_per_audio_token_flex?: number | null;
/** Input Cost Per Audio Token Priority */
input_cost_per_audio_token_priority?: number | null;
/** Input Cost Per Character */
input_cost_per_character?: number | null;
/** Input Cost Per Character Above 128K Tokens */
@ -50568,6 +50578,12 @@ export interface components {
cache_creation_input_token_cost_ultrafast?: number | null;
/** Cache Read Input Audio Token Cost */
cache_read_input_audio_token_cost?: number | null;
/** Cache Read Input Audio Token Cost Batches */
cache_read_input_audio_token_cost_batches?: number | null;
/** Cache Read Input Audio Token Cost Flex */
cache_read_input_audio_token_cost_flex?: number | null;
/** Cache Read Input Audio Token Cost Priority */
cache_read_input_audio_token_cost_priority?: number | null;
/** Cache Read Input Image Token Cost */
cache_read_input_image_token_cost?: number | null;
/** Cache Read Input Token Cost */
@ -50647,6 +50663,10 @@ export interface components {
input_cost_per_audio_token?: number | null;
/** Input Cost Per Audio Token Batches */
input_cost_per_audio_token_batches?: number | null;
/** Input Cost Per Audio Token Flex */
input_cost_per_audio_token_flex?: number | null;
/** Input Cost Per Audio Token Priority */
input_cost_per_audio_token_priority?: number | null;
/** Input Cost Per Character */
input_cost_per_character?: number | null;
/** Input Cost Per Character Above 128K Tokens */