Merge remote-tracking branch 'origin/main' into HEAD

# Conflicts:
#	litellm/responses/utils.py
This commit is contained in:
mateo-berri 2026-10-24 09:20:18 -07:00
commit 9dfcb67470
33 changed files with 1646 additions and 302 deletions

View file

@ -2268,6 +2268,8 @@ def response_cost_calculator(
### VERTEX LOCATION ###
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
) -> float:
"""
Returns
@ -2296,6 +2298,8 @@ def response_cost_calculator(
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
custom_cost_per_token=custom_cost_per_token,
custom_cost_per_second=custom_cost_per_second,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,
@ -2879,6 +2883,7 @@ def _batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"cache_read_input_audio_token_cost_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
@ -3012,6 +3017,7 @@ def _batch_prompt_cost(
) -> float:
details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
audio_tokens, image_tokens, video_tokens = (
details["audio_tokens"],
@ -3026,7 +3032,9 @@ def _batch_prompt_cost(
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ cached_tokens * (cache_read_rate or 0.0)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
+ cached_audio_tokens
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
+ written_tokens * (cache_creation_rate or 0.0)
)

View file

@ -161,6 +161,7 @@ from litellm.types.utils import (
CachingDetails,
CallTypes,
CostBreakdown,
CostPerToken,
CustomPricingLiteLLMParams,
DynamicPromptManagementParamLiteral,
EmbeddingResponse,
@ -622,6 +623,8 @@ class Logging(LiteLLMLoggingBaseClass):
prometheusLogger, \
slack_app
custom_pricing: bool = False
custom_cost_per_token: CostPerToken | None = None
custom_cost_per_second: float | None = None
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
litellm_request_debug: bool = False
@ -2097,6 +2100,8 @@ class Logging(LiteLLMLoggingBaseClass):
"call_type": self.call_type,
"optional_params": self.optional_params,
"custom_pricing": custom_pricing,
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
"prompt": prompt,
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,

View file

@ -5,30 +5,42 @@ Shared by provider cost calculators (e.g. Dashscope) and the proxy budget
reservation logic so neither has to depend on the other.
"""
from typing import Final
from collections.abc import Mapping, Sequence
from typing import Final, SupportsFloat
from pydantic import TypeAdapter, ValidationError
_TIER_RANGE: Final = TypeAdapter(tuple[float, float])
def _coerce_cost_per_token(value: float | str | None) -> float:
def _coerce_cost_per_token(value: object) -> float:
"""
Coerce a per-token cost into a float.
Model cost values loaded from YAML config may arrive as strings (e.g.
scientific notation like "4e-07"), which would break arithmetic.
"""
if value is None:
return 0.0
if isinstance(value, str):
try:
return float(value)
except ValueError:
return 0.0
return float(value)
if isinstance(value, SupportsFloat):
return float(value)
return 0.0
def _tier_range(tier: Mapping[str, object]) -> tuple[float, float] | None:
try:
return _TIER_RANGE.validate_python(tier.get("range"))
except ValidationError:
return None
def select_tier_for_input(
tiered_pricing: list[dict],
tiered_pricing: Sequence[Mapping[str, object]],
input_tokens: int,
) -> dict | None:
) -> Mapping[str, object] | None:
"""
Select the pricing tier for a request based on its total input token count.
@ -44,19 +56,21 @@ def select_tier_for_input(
if not tiered_pricing or input_tokens <= 0:
return None
sorted_tiers: Final = sorted(tiered_pricing, key=lambda t: t.get("range", [0, 0])[0])
valid_tiers: Final = [tier for tier in sorted_tiers if len(tier.get("range", [])) == 2]
if not valid_tiers:
ranged_tiers: Final = sorted(
((bounds, tier) for tier in tiered_pricing if (bounds := _tier_range(tier)) is not None),
key=lambda ranged: ranged[0][0],
)
if not ranged_tiers:
return None
matching: Final = [tier for tier in valid_tiers if tier["range"][0] < input_tokens <= tier["range"][1]]
matching: Final = tuple(tier for (start, end), tier in ranged_tiers if start < input_tokens <= end)
if matching:
return matching[0]
return valid_tiers[-1]
return ranged_tiers[-1][1]
def tier_rate(
tier: dict,
tier: Mapping[str, object],
cost_key: str,
fallback_cost_key: str | None = None,
) -> float:
@ -68,4 +82,6 @@ def tier_rate(
primary: Final = tier.get(cost_key)
if primary is not None:
return _coerce_cost_per_token(primary)
if fallback_cost_key is None:
return 0.0
return _coerce_cost_per_token(tier.get(fallback_cost_key, 0))

View file

@ -3,7 +3,7 @@
import re
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone, tzinfo
from types import MappingProxyType
from typing import Final, Literal, TypedDict, cast
@ -52,6 +52,22 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency)
_DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields)
_BASE_CACHE_WRITE_RATE_KEY: Final = "cache_creation_input_token_cost"
_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
{
"5m": _BASE_CACHE_WRITE_RATE_KEY,
"30m": _BASE_CACHE_WRITE_RATE_KEY,
"1hr": "cache_creation_input_token_cost_above_1hr",
"12hr": "cache_creation_input_token_cost_above_12hr",
}
)
_TTL_PRICED_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
{ttl: key for ttl, key in _CACHE_WRITE_RATE_KEYS.items() if key != _BASE_CACHE_WRITE_RATE_KEY}
)
_IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
"input_cost_per_token",
"output_cost_per_token",
@ -356,7 +372,21 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide
)
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict[str, object] | None:
@dataclass(frozen=True, slots=True)
class TokenRates:
input_rate: float
output_rate: float
cache_read_rate: float
cache_creation_rate: float
reasoning_rate: float | None
cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
@property
def billed_reasoning_rate(self) -> float:
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, object] | None:
tiered_pricing: Final = model_info.get("tiered_pricing")
if not isinstance(tiered_pricing, list) or not tiered_pricing:
return None
@ -376,7 +406,7 @@ def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | N
return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token")
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float, float, float, float, float] | None:
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None:
"""
Resolve the base rates from a model's ``tiered_pricing`` table, if it has one.
@ -391,22 +421,20 @@ def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float,
if tier is None:
return None
cache_creation_cost: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
completion_cost: Final = (
tier_rate(tier, "output_cost_per_token")
if "output_cost_per_token" in tier
else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0
)
return (
tier_rate(tier, "input_cost_per_token"),
completion_cost,
cache_creation_cost,
tier_rate(tier, "cache_creation_input_token_cost_above_1hr")
if "cache_creation_input_token_cost_above_1hr" in tier
and tier["cache_creation_input_token_cost_above_1hr"] is not None
else cache_creation_cost,
tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
rates: Final = TokenRates(
input_rate=tier_rate(tier, "input_cost_per_token"),
output_rate=completion_cost,
cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
cache_creation_rate=cache_creation_rate,
reasoning_rate=None,
)
return rates, tier
def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool:
@ -532,19 +560,6 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None =
return False
@dataclass(frozen=True, slots=True)
class TokenRates:
input_rate: float
output_rate: float
cache_read_rate: float
cache_creation_rate: float
reasoning_rate: float | None
@property
def billed_reasoning_rate(self) -> float:
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
def resolve_explicit_cache_read_cost_rate(
model_info: ModelInfo,
usage: Usage,
@ -649,7 +664,8 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
if off_peak is None:
return rates
off_peak_reasoning_rate: Final = _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token"))
return TokenRates(
return replace(
rates,
input_rate=_off_peak_rate(off_peak, "input_cost_per_token", rates.input_rate),
output_rate=_off_peak_rate(off_peak, "output_cost_per_token", rates.output_rate),
cache_read_rate=_off_peak_rate(off_peak, "cache_read_input_token_cost", rates.cache_read_rate),
@ -661,32 +677,28 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
def _apply_off_peak_to_base_costs(
model_info: ModelInfo,
current_time: datetime | None,
base_costs: tuple[float, float, float, float | None, float],
) -> tuple[float, float, float, float, float]:
"""Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a
present one passes through untouched and an absent one resolves to the applied
cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate.
base_rates: TokenRates,
tier: Mapping[str, object],
) -> TokenRates:
"""Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no
field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier
that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to
_resolve_billed_reasoning_rate.
"""
prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
rates: Final = apply_off_peak_pricing(
model_info,
current_time,
TokenRates(
input_rate=prompt,
output_rate=completion,
cache_read_rate=cache_read,
cache_creation_rate=cache_creation,
reasoning_rate=None,
rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates)
return replace(
rates,
cache_creation_rates_by_ttl=MappingProxyType(
{
ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key)
for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()
}
),
)
return (
rates.input_rate,
rates.output_rate,
rates.cache_creation_rate,
rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr,
rates.cache_read_rate,
)
def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str:
return preferred_key if model_info.get(preferred_key) is not None else fallback_key
def _get_token_base_cost(
@ -696,7 +708,7 @@ def _get_token_base_cost(
current_time: datetime | None = None,
*,
threshold_is_inclusive: bool = False,
) -> tuple[float, float, float, float, float]:
) -> TokenRates:
"""
Return prompt cost, completion cost, and cache costs for a given model and usage.
@ -709,15 +721,12 @@ def _get_token_base_cost(
An absent cache-creation or cache-read rate always resolves to the resolved input
rate, the way the tiered table and custom deployment pricing already do, since a
provider that publishes no cache price bills cached tokens as ordinary input. An
absent 1h write rate resolves to the cache-creation rate, off-peak included. An
absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An
explicit 0.0 stays a real price for all of them.
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage)
if tiered_base_costs is not None:
return _apply_off_peak_to_base_costs(model_info, current_time, tiered_base_costs)
return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs)
# Get service tier aware cost keys
input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier)
@ -734,13 +743,13 @@ def _get_token_base_cost(
# For image generation models that don't have output_cost_per_token,
# use output_cost_per_image_token as the base cost (all output tokens are image tokens)
if completion_base_cost == 0.0 or completion_base_cost is None:
if completion_base_cost == 0.0:
output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None)
if output_image_cost is not None:
completion_base_cost = output_image_cost
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None)
cache_creation_cost_above_1hr = get_cost_per_unit(
model_info, "cache_creation_input_token_cost_above_1hr", default_value=None
cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType(
{ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()}
)
cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None)
@ -811,14 +820,6 @@ def _get_token_base_cost(
if service_tier
else f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
)
cache_creation_1hr_tiered_key = (
get_service_tier_cost_key(
f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens",
service_tier,
)
if service_tier
else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
)
cache_read_tiered_key = (
get_service_tier_cost_key(
f"cache_read_input_token_cost_above_{threshold_str}_tokens",
@ -830,10 +831,19 @@ def _get_token_base_cost(
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost)
cache_creation_cost_above_1hr = get_cost_per_unit(
model_info, cache_creation_1hr_tiered_key, cache_creation_cost_above_1hr
cache_creation_keys_by_ttl = MappingProxyType(
{
ttl: _first_present_key(
model_info,
get_service_tier_cost_key(
f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens",
service_tier,
),
key,
)
for ttl, key in cache_creation_keys_by_ttl.items()
}
)
cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
except (IndexError, ValueError):
continue
@ -849,16 +859,26 @@ def _get_token_base_cost(
resolved_cache_creation_cost: Final = (
input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
)
return _apply_off_peak_to_base_costs(
base_rates: Final = apply_off_peak_pricing(
model_info,
current_time,
(
prompt_base_cost,
completion_base_cost,
resolved_cache_creation_cost,
cache_creation_cost_above_1hr,
resolved_cache_read_cost,
TokenRates(
input_rate=prompt_base_cost,
output_rate=completion_base_cost,
cache_read_rate=resolved_cache_read_cost,
cache_creation_rate=resolved_cache_creation_cost,
reasoning_rate=None,
),
)
return replace(
base_rates,
cache_creation_rates_by_ttl=MappingProxyType(
{
ttl: base_rates.cache_creation_rate
if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None
else rate
for ttl, key in cache_creation_keys_by_ttl.items()
}
),
)
@ -975,25 +995,28 @@ def resolve_image_model_info(model: str, custom_llm_provider: str, model_info: M
def calculate_cache_writing_cost(
cache_creation_tokens: int,
cache_creation_token_details: CacheCreationTokenDetails | None,
cache_creation_cost_above_1hr: float,
cache_creation_cost: float,
cache_creation_cost_by_ttl: Mapping[str, float],
) -> float:
"""
Adjust cost of cache creation tokens based on the cache creation token details.
Bill cache writes per lifetime: Anthropic reports 5m and 1h counts, OpenAI reports 30m and 12h counts.
Writes no lifetime count covers, such as a lifetime a provider adds later, bill at the base write rate.
"""
total_cost: float = 0.0
if cache_creation_token_details is not None:
# get the number of 5m and 1h cache creation tokens
cache_creation_tokens_5m: Final = cache_creation_token_details.ephemeral_5m_input_tokens
cache_creation_tokens_1h: Final = cache_creation_token_details.ephemeral_1h_input_tokens
# add the number of 5m and 1h cache creation tokens to the cache creation tokens
total_cost += cache_creation_tokens_5m * cache_creation_cost if cache_creation_tokens_5m is not None else 0.0
total_cost += (
cache_creation_tokens_1h * cache_creation_cost_above_1hr if cache_creation_tokens_1h is not None else 0.0
)
else:
total_cost += cache_creation_tokens * cache_creation_cost
return total_cost
if cache_creation_token_details is None:
return cache_creation_tokens * cache_creation_cost
tokens_by_ttl: Final[Mapping[str, int | None]] = MappingProxyType(
{
"5m": cache_creation_token_details.ephemeral_5m_input_tokens,
"30m": cache_creation_token_details.ephemeral_30m_input_tokens,
"1hr": cache_creation_token_details.ephemeral_1h_input_tokens,
"12hr": cache_creation_token_details.ephemeral_12h_input_tokens,
}
)
unattributed_tokens: Final = max(cache_creation_tokens - sum(t or 0 for t in tokens_by_ttl.values()), 0)
return unattributed_tokens * cache_creation_cost + sum(
(tokens or 0) * cache_creation_cost_by_ttl.get(ttl, cache_creation_cost)
for ttl, tokens in tokens_by_ttl.items()
)
class PromptTokensDetailsResult(TypedDict):
@ -1178,7 +1201,7 @@ def _calculate_input_cost(
prompt_base_cost: float,
cache_read_cost: float,
cache_creation_cost: float,
cache_creation_cost_above_1hr: float,
cache_creation_cost_by_ttl: Mapping[str, float],
service_tier: str | None = None,
) -> float:
"""
@ -1248,8 +1271,8 @@ def _calculate_input_cost(
prompt_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"],
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_by_ttl=cache_creation_cost_by_ttl,
)
### CHARACTER COST
@ -1510,34 +1533,31 @@ def generic_cost_per_token(
)
billing_time: Final = current_time if current_time is not None else current_billing_time()
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
) = _get_token_base_cost(
base_costs: Final = _get_token_base_cost(
model_info=resolved_model_info,
usage=usage,
service_tier=service_tier,
current_time=billing_time,
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
)
completion_base_cost: Final = base_costs.output_rate
resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate(
model_info=resolved_model_info,
usage=usage,
current_time=billing_time,
service_tier=service_tier,
)
effective_cache_read_cost: Final = cache_read_cost if resolved_cache_read_cost is None else resolved_cache_read_cost
effective_cache_read_cost: Final = (
base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost
)
prompt_cost = _calculate_input_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=resolved_model_info,
prompt_base_cost=prompt_base_cost,
prompt_base_cost=base_costs.input_rate,
cache_read_cost=effective_cache_read_cost,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=base_costs.cache_creation_rate,
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
service_tier=service_tier,
)
@ -1640,7 +1660,7 @@ class BilledTokenRates:
cache_read_input_token_cost: float
cache_read_input_audio_token_cost: float
cache_creation_input_token_cost: float
cache_creation_input_token_cost_above_1hr: float
cache_creation_input_token_cost_by_ttl: Mapping[str, float]
output_cost_per_reasoning_token: float
def scaled(self, multiplier: float) -> "BilledTokenRates":
@ -1652,7 +1672,9 @@ class BilledTokenRates:
cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier,
cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier,
cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier,
cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier,
cache_creation_input_token_cost_by_ttl=MappingProxyType(
{ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()}
),
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
)
@ -1702,7 +1724,7 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat
cache_read_input_token_cost=cache_read_rate,
cache_read_input_audio_token_cost=cache_read_rate,
cache_creation_input_token_cost=cache_creation_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_rate,
cache_creation_input_token_cost_by_ttl=MappingProxyType({}),
output_cost_per_reasoning_token=output_rate,
)
@ -1717,13 +1739,7 @@ def _cost_map_billed_rates(
current_time: datetime | None,
) -> BilledTokenRates:
billing_time: Final = current_time if current_time is not None else current_billing_time()
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost_rate,
cache_creation_cost_above_1hr_rate,
cache_read_cost_rate,
) = _get_token_base_cost(
base_costs: Final = _get_token_base_cost(
model_info=model_info,
usage=usage,
service_tier=service_tier,
@ -1737,14 +1753,14 @@ def _cost_map_billed_rates(
service_tier=service_tier,
)
effective_cache_read_cost_rate: Final = (
cache_read_cost_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
)
reasoning_rate: Final = _resolve_billed_reasoning_rate(
model_info=model_info,
usage=usage,
service_tier=service_tier,
completion_base_cost=completion_base_cost,
completion_base_cost=base_costs.output_rate,
current_time=billing_time,
)
audio_cache_read_rate: Final = get_cost_per_unit(
@ -1759,14 +1775,14 @@ def _cost_map_billed_rates(
* get_provider_specific_speed_multiplier(model_info=model_info, usage=usage)
)
return BilledTokenRates(
input_cost_per_token=prompt_base_cost,
output_cost_per_token=completion_base_cost,
input_cost_per_token=base_costs.input_rate,
output_cost_per_token=base_costs.output_rate,
cache_read_input_token_cost=effective_cache_read_cost_rate,
cache_read_input_audio_token_cost=(
audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate
),
cache_creation_input_token_cost=cache_creation_cost_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
cache_creation_input_token_cost=base_costs.cache_creation_rate,
cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
output_cost_per_reasoning_token=reasoning_rate,
).scaled(multiplier)
@ -1854,8 +1870,8 @@ def get_token_type_cost_breakdown(
else calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
cache_creation_cost_above_1hr=rates.cache_creation_input_token_cost_above_1hr,
cache_creation_cost=rates.cache_creation_input_token_cost,
cache_creation_cost_by_ttl=rates.cache_creation_input_token_cost_by_ttl,
)
)
return TokenTypeCostBreakdown(
@ -1886,7 +1902,7 @@ def calculate_prompt_caching_savings(
biller saw them rather than at the later spend write.
"""
model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider)
prompt_base_cost, _, cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost = _get_token_base_cost(
base_costs: Final = _get_token_base_cost(
model_info=model_info_with_cache_read_default,
usage=usage,
service_tier=service_tier,
@ -1899,9 +1915,11 @@ def calculate_prompt_caching_savings(
current_time=billed_at,
service_tier=service_tier,
)
effective_cache_read_cost: Final = cache_read_cost if provider_cache_read_cost is None else provider_cache_read_cost
write_rate: Final = cache_creation_cost or prompt_base_cost
write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate
prompt_base_cost: Final = base_costs.input_rate
effective_cache_read_cost: Final = (
base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost
)
write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0)
cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0)
@ -1909,7 +1927,9 @@ def calculate_prompt_caching_savings(
cache_creation_details: Final = (
CacheCreationTokenDetails(
ephemeral_5m_input_tokens=max(details.ephemeral_5m_input_tokens or 0, 0),
ephemeral_30m_input_tokens=max(details.ephemeral_30m_input_tokens or 0, 0),
ephemeral_1h_input_tokens=max(details.ephemeral_1h_input_tokens or 0, 0),
ephemeral_12h_input_tokens=max(details.ephemeral_12h_input_tokens or 0, 0),
)
if details is not None
else None
@ -1918,8 +1938,10 @@ def calculate_prompt_caching_savings(
write_premium: Final = calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_details,
cache_creation_cost_above_1hr=write_rate_1h - prompt_base_cost,
cache_creation_cost=write_rate - prompt_base_cost,
cache_creation_cost_by_ttl=MappingProxyType(
{ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()}
),
)
uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(
model_info, vertex_location

View file

@ -5,6 +5,7 @@ from typing import Final
from pydantic import TypeAdapter, ValidationError
from typing_extensions import assert_never
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
@ -16,6 +17,8 @@ ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total"
_TEXT_INPUT_RATE: Final = "input_cost_per_token"
_AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token"
_CACHE_READ_INPUT_RATE: Final = "cache_read_input_token_cost"
_CACHE_READ_AUDIO_INPUT_RATE: Final = "cache_read_input_audio_token_cost"
_TEXT_OUTPUT_RATE: Final = "output_cost_per_token"
_AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token"
_RATE_KEY_MARKERS: Final = ("cost", "pricing")
@ -33,19 +36,38 @@ def _tokens(value: object) -> int:
def used_pricing_keys(usage: Usage) -> tuple[str, ...]:
prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details)
prompt_details: Final = parse_prompt_tokens_details(usage)
prompt_audio: Final = prompt_details["audio_tokens"]
completion_audio: Final = _audio_tokens(usage.completion_tokens_details)
prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio
prompt_text: Final = max(
_tokens(usage.prompt_tokens)
- prompt_details["cache_hit_tokens"]
- prompt_audio
- prompt_details["image_tokens"]
- prompt_details["video_tokens"],
0,
)
cached_text: Final = max(prompt_details["cache_hit_tokens"] - prompt_details["cache_hit_audio_tokens"], 0)
completion_text: Final = _tokens(usage.completion_tokens) - completion_audio
components: Final = (
(_TEXT_INPUT_RATE, prompt_text),
(_AUDIO_INPUT_RATE, prompt_audio),
(_CACHE_READ_INPUT_RATE, cached_text),
(_CACHE_READ_AUDIO_INPUT_RATE, prompt_details["cache_hit_audio_tokens"]),
(_TEXT_OUTPUT_RATE, completion_text),
(_AUDIO_OUTPUT_RATE, completion_audio),
)
return tuple(key for key, count in components if count > 0)
def _effective_pricing_key(key: str, pricing_entry: Mapping[str, object]) -> str:
if key == _CACHE_READ_AUDIO_INPUT_RATE and pricing_entry.get(key) is None:
return _CACHE_READ_INPUT_RATE if pricing_entry.get(_CACHE_READ_INPUT_RATE) is not None else _TEXT_INPUT_RATE
if key == _CACHE_READ_INPUT_RATE and pricing_entry.get(key) is None:
return _TEXT_INPUT_RATE
return key
def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None:
try:
return _NESTED_PRICING.validate_python(value)
@ -89,7 +111,8 @@ def diagnose_zero_cost(
pricing_entry: Mapping[str, object],
calculation_failed: bool,
) -> StandardLoggingZeroCostDiagnostic | None:
used_keys: Final = used_pricing_keys(usage)
effective_keys: Final = tuple(_effective_pricing_key(key, pricing_entry) for key in used_pricing_keys(usage))
used_keys: Final = tuple(key for index, key in enumerate(effective_keys) if key not in effective_keys[:index])
if not used_keys:
return None
missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None)

View file

@ -7,6 +7,7 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate.
See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
"""
from collections.abc import Mapping
from dataclasses import dataclass, replace
from datetime import datetime
from typing import Final
@ -75,7 +76,7 @@ def _flat_rates(model_info: ModelInfo) -> TokenRates:
)
def _tier_rates(model_info: ModelInfo, tier: dict) -> TokenRates:
def _tier_rates(model_info: ModelInfo, tier: Mapping[str, object]) -> TokenRates:
# A tier that declares output rates keeps the request on them, all-or-nothing. A tier table
# spelling out only input rates would serve every completion for free, so there the model's
# own output rates stand in

View file

@ -42,6 +42,7 @@ from litellm.llms.custom_httpx.http_handler import (
get_httpx_client,
)
from litellm.types.llms.anthropic import AnthropicThinkingParam
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
from litellm.types.llms.openai import (
AllMessageValues,
@ -2364,26 +2365,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
elif modality == "DOCUMENT":
cached_text_tokens = (cached_text_tokens or 0) + token_count
## Calculate non-cached tokens by subtracting cached from total (per modality)
## This is necessary because promptTokensDetails includes both cached and non-cached tokens
## See: https://github.com/BerriAI/litellm/issues/18750
if cached_text_tokens is not None and prompt_text_tokens is not None:
# Explicit caching: subtract cached tokens per modality from cacheTokensDetails
prompt_text_tokens = prompt_text_tokens - cached_text_tokens
elif (
if (
cached_tokens is not None
and prompt_text_tokens is not None
and cached_text_tokens is None
and "cacheTokensDetails" not in usage_metadata
):
# Implicit caching: only cachedContentTokenCount is provided (no cacheTokensDetails)
# Subtract from text tokens since implicit caching is primarily for text content
# See: https://github.com/BerriAI/litellm/issues/16341
prompt_text_tokens = prompt_text_tokens - cached_tokens
if cached_audio_tokens is not None and prompt_audio_tokens is not None:
prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens
if cached_image_tokens is not None and prompt_image_tokens is not None:
prompt_image_tokens = prompt_image_tokens - cached_image_tokens
if cached_video_tokens is not None and prompt_video_tokens is not None:
prompt_video_tokens = prompt_video_tokens - cached_video_tokens
@ -2404,9 +2392,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
billable_text_tokens: Final = (
prompt_text_tokens if prompt_text_tokens is None else prompt_text_tokens + billable_tool_use_prompt_tokens
)
cached_tokens_details_fields: Final = MappingProxyType(
{
key: token_count
for key, token_count in (
("text_tokens", cached_text_tokens),
("audio_tokens", cached_audio_tokens),
("image_tokens", cached_image_tokens),
)
if token_count is not None
}
)
cached_tokens_details: Final = (
CachedTokensDetails.model_validate(cached_tokens_details_fields) if cached_tokens_details_fields else None
)
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cached_tokens,
cached_tokens_details=cached_tokens_details,
audio_tokens=prompt_audio_tokens,
text_tokens=billable_text_tokens,
image_tokens=prompt_image_tokens,

View file

@ -31137,11 +31137,15 @@
},
"gemini/gemini-2.5-flash": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 3e-08,
"cache_read_input_token_cost_batches": 3e-08,
"cache_read_input_token_cost_flex": 3e-08,
"cache_read_input_token_cost_priority": 5.4e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 3e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31466,11 +31470,15 @@
},
"gemini/gemini-2.5-flash-lite": {
"cache_read_input_audio_token_cost": 3e-08,
"cache_read_input_audio_token_cost_batches": 3e-08,
"cache_read_input_audio_token_cost_flex": 3e-08,
"cache_read_input_audio_token_cost_priority": 5.4e-08,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_batches": 1e-08,
"cache_read_input_token_cost_flex": 1e-08,
"cache_read_input_token_cost_priority": 1.8e-08,
"input_cost_per_audio_token": 3e-07,
"input_cost_per_audio_token_flex": 1.5e-07,
"input_cost_per_token": 1e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31760,12 +31768,16 @@
},
"gemini/gemini-3.1-flash-lite": {
"cache_read_input_audio_token_cost": 5e-08,
"cache_read_input_audio_token_cost_batches": 2.5e-08,
"cache_read_input_audio_token_cost_flex": 2.5e-08,
"cache_read_input_audio_token_cost_priority": 9e-08,
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"deprecation_date": "2027-05-07",
"input_cost_per_audio_token": 5e-07,
"input_cost_per_audio_token_flex": 2.5e-07,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"input_cost_per_token_flex": 1.25e-07,
@ -31886,10 +31898,14 @@
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 5e-08,
"cache_read_input_token_cost_batches": 5e-08,
"cache_read_input_token_cost_flex": 5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -36795,6 +36811,12 @@
},
"gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_12hr": 2.5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001,
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-05,
"cache_creation_input_token_cost_above_12hr_priority": 5e-05,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
@ -36880,6 +36902,12 @@
},
"gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -36957,6 +36985,12 @@
},
"gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_12hr": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06,
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-07,
"cache_creation_input_token_cost_above_12hr_priority": 5e-07,
"cache_creation_input_token_cost_decisions": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
@ -37043,6 +37077,12 @@
},
"gpt-5.6": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_12hr": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37117,6 +37157,12 @@
},
"gpt-5.6-sol": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_12hr": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37193,6 +37239,12 @@
},
"gpt-5.6-terra": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -37268,6 +37320,12 @@
},
"gpt-5.6-luna": {
"cache_creation_input_token_cost": 2.5e-07,
"cache_creation_input_token_cost_above_12hr": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_priority": 1e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
@ -45847,6 +45905,7 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -45872,6 +45931,7 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -49587,6 +49647,7 @@
"supports_vision": false
},
"us-gov.openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -49606,6 +49667,7 @@
"supports_tool_choice": true
},
"us-gov.openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -60156,6 +60218,7 @@
}
},
"bedrock_mantle/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7.5e-08,
"input_cost_per_token_flex": 7.5e-08,
@ -60181,6 +60244,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3.5e-08,
"input_cost_per_token_flex": 3.5e-08,
@ -60206,6 +60270,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7e-08,
"input_cost_per_token_flex": 7e-08,
@ -60229,6 +60294,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
@ -68080,6 +68146,7 @@
"supports_vision": false
},
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68099,6 +68166,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68377,6 +68445,7 @@
"supports_vision": false
},
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68396,6 +68465,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68784,6 +68854,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68803,6 +68874,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -68943,6 +69015,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68962,6 +69035,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -74917,6 +74991,12 @@
},
"gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,

View file

@ -22858,6 +22858,25 @@
"title": "Body_image_edit_api_openai_deployments__model__images_edits_post",
"type": "object"
},
"CacheWriteTokensDetails": {
"additionalProperties": true,
"properties": {
"ttl_12h_tokens": {
"title": "Ttl 12H Tokens",
"type": "integer"
},
"ttl_30m_tokens": {
"title": "Ttl 30M Tokens",
"type": "integer"
}
},
"required": [
"ttl_30m_tokens",
"ttl_12h_tokens"
],
"title": "CacheWriteTokensDetails",
"type": "object"
},
"CachedTokensDetails": {
"properties": {
"audio_tokens": {
@ -24352,6 +24371,27 @@
],
"title": "Audio Tokens"
},
"cache_write_tokens": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
],
"title": "Cache Write Tokens"
},
"cache_write_tokens_details": {
"anyOf": [
{
"$ref": "#/components/schemas/CacheWriteTokensDetails"
},
{
"type": "null"
}
]
},
"cached_tokens": {
"default": 0,
"title": "Cached Tokens",

View file

@ -3663,18 +3663,20 @@ class LiteLLMCompletionResponsesConfig:
if count is not None
}
)
response_usage.input_tokens_details = InputTokensDetails(
cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
text_tokens=prompt_details.text_tokens,
audio_tokens=prompt_details.audio_tokens,
image_tokens=prompt_details.image_tokens,
video_tokens=prompt_details.video_tokens,
cached_tokens_details=(
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),
**cache_write_extra,
**cache_type_extra,
**grounding_request_counts,
response_usage.input_tokens_details = InputTokensDetails.model_validate(
{
"cached_tokens": prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
"text_tokens": prompt_details.text_tokens,
"audio_tokens": prompt_details.audio_tokens,
"image_tokens": prompt_details.image_tokens,
"video_tokens": prompt_details.video_tokens,
"cached_tokens_details": (
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
),
**cache_write_extra,
**cache_type_extra,
**grounding_request_counts,
}
)
# Translate completion_tokens_details to output_tokens_details

View file

@ -6,10 +6,10 @@ from contextlib import contextmanager
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
import httpx
from pydantic import BaseModel, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from typing_extensions import (
TypeIs, # noqa: TID251 # narrows a client input item before its mark is read
assert_never,
@ -87,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
from litellm.types.responses.main import *
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params
from litellm.types.utils import (
ADDRESSED_RESPONSE_ID_FIELD,
CallTypes,
CostPerToken,
CustomPricingLiteLLMParams,
all_litellm_params,
)
from litellm.utils import (
ProviderConfigManager,
client,
@ -547,6 +553,47 @@ async def aresponses_api_with_mcp(
return response
def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]:
return MappingProxyType(
litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True)
)
class _DeploymentModelInfo(BaseModel):
model_config = ConfigDict(frozen=True)
id: object = None
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
)
def _is_router_deployment(deployment_model_info: object) -> bool:
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
return None
return _COST_PER_TOKEN.validate_python(
litellm_params.model_dump(
include={
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
},
exclude_none=True,
)
)
def _bridges_to_chat_completions(
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
) -> bool:
@ -1674,6 +1721,7 @@ def responses(
optional_params=dict(responses_api_request_params),
litellm_params={
**responses_api_request_params,
**_per_call_custom_pricing(litellm_params),
"aresponses": _is_async,
"litellm_call_id": litellm_call_id,
"model_info": kwargs.get("model_info"),
@ -1684,6 +1732,9 @@ def responses(
custom_llm_provider=custom_llm_provider,
)
if not _is_router_deployment(deployment_model_info):
litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params)
litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second
# Decode any litellm-encoded encrypted-content item IDs back to their original IDs
input = with_compaction_items_as_summaries(
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))

View file

@ -15,6 +15,7 @@ from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
from litellm.types.llms.openai import (
AllMessageValues,
InputTokensDetails,
OutputTokensDetails,
ResponseAPIUsage,
ResponseInputParam,
@ -25,6 +26,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.responses.main import DecodedResponseId
from litellm.types.utils import (
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
SpecialEnums,
@ -47,6 +49,17 @@ def _output_token_detail(details: object, field: str) -> int | None:
return value if isinstance(value, int) else None
def _cache_write_lifetimes(cache_write_tokens_details: object) -> CacheCreationTokenDetails | None:
"""OpenAI Responses splits `cache_write_tokens` into `ttl_30m_tokens` and `ttl_12h_tokens`."""
ttl_30m_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_30m_tokens")
ttl_12h_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_12h_tokens")
if ttl_30m_tokens is None and ttl_12h_tokens is None:
return None
return CacheCreationTokenDetails(
ephemeral_30m_input_tokens=ttl_30m_tokens, ephemeral_12h_input_tokens=ttl_12h_tokens
)
def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a list is a Sequence of anything
return isinstance(value, list)
@ -59,7 +72,11 @@ def _is_object_dict(
def _prompt_tokens_details_wrapper(details: object) -> PromptTokensDetailsWrapper:
if _is_object_dict(details):
return PromptTokensDetailsWrapper(**details)
input_details: Final = InputTokensDetails.model_validate(details)
return PromptTokensDetailsWrapper(
**input_details.model_dump(exclude={"cache_write_tokens_details"}),
cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details),
)
return PromptTokensDetailsWrapper(
cached_tokens=getattr(details, "cached_tokens", None),
audio_tokens=getattr(details, "audio_tokens", None),
@ -68,6 +85,7 @@ def _prompt_tokens_details_wrapper(details: object) -> PromptTokensDetailsWrappe
cached_tokens_details=getattr(details, "cached_tokens_details", None),
video_tokens=getattr(details, "video_tokens", None),
cache_write_tokens=getattr(details, "cache_write_tokens", None),
cache_creation_token_details=_cache_write_lifetimes(getattr(details, "cache_write_tokens_details", None)),
cache_type=getattr(details, "cache_type", None),
web_search_requests=getattr(details, "web_search_requests", None),
google_maps_grounding_requests=getattr(details, "google_maps_grounding_requests", None),

View file

@ -14186,6 +14186,43 @@ class Router:
deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params)
)
@staticmethod
def _deployment_declares_no_vision(deployment: DeploymentTypedDict, group: str) -> bool:
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
declared: Final = model_info.get("supports_vision")
if declared is not None:
return declared is False
raw_params: Final = plain_deployment.get("litellm_params") or {}
params: Final = cast(Mapping[str, object], raw_params) # cast-ok: router deployments are plain dicts
if not params:
return False
base_model: Final = model_info.get("base_model") or params.get("base_model")
try:
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
return litellm.utils.is_vision_explicitly_disabled(
base_model if isinstance(base_model, str) else model, custom_llm_provider
)
except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
verbose_router_logger.debug(
"litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e
)
return False
@staticmethod
def _request_carries_image(
messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is
request_kwargs: Mapping[str, object],
) -> bool:
from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
structured: Final = resolve_structured_messages(
messages=cast(list[dict[str, object]] | None, messages), # cast-ok: chat messages are plain dicts
request_kwargs=cast(dict[str, object], request_kwargs), # cast-ok: the resolver only reads the mapping
)
return bool(structured) and request_contains_image_content(structured)
@staticmethod
def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]:
requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs))
@ -14212,6 +14249,7 @@ class Router:
- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
- filter models above rpm limits
- if region given, filter out models not in that region / unknown region
- request carries an image and the deployment declares supports_vision false, unless every candidate does
- [TODO] function call and model doesn't support function calling
"""
@ -14221,7 +14259,7 @@ class Router:
# We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request)
_returned_deployments = list(healthy_deployments)
invalid_model_indices: Final = set() # Use set for O(1) membership checks
invalid_model_indices: Final = set[int]() # Use set for O(1) membership checks
_context_window_error = False
_potential_error_str = ""
@ -14371,9 +14409,20 @@ class Router:
model=model,
llm_provider="",
)
if len(excluded_indices) > 0:
remaining: Final = tuple(
(idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices
)
text_only_indices: Final[frozenset[int]] = (
frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model))
if remaining and self._request_carries_image(messages, request_settings)
else frozenset()
)
dropped_indices: Final[frozenset[int]] = excluded_indices | (
text_only_indices if len(text_only_indices) < len(remaining) else frozenset()
)
if len(dropped_indices) > 0:
# Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops)
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices]
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices]
return _returned_deployments

View file

@ -683,12 +683,12 @@ class ChatCompletionCachedContent(TypedDict):
class PromptCacheBreakpoint(TypedDict):
mode: ReadOnly[Literal["explicit"]]
ttl: NotRequired[ReadOnly[Literal["30m"]]]
ttl: NotRequired[ReadOnly[Literal["30m", "12h"]]]
class PromptCacheOptions(TypedDict, total=False):
mode: ReadOnly[Literal["implicit", "explicit"]]
ttl: ReadOnly[Literal["30m"]]
ttl: ReadOnly[Literal["30m", "12h"]]
class ChatCompletionThinkingBlock(TypedDict, total=False):
@ -1382,10 +1382,23 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
model_config = {"extra": "allow"}
class CacheWriteTokensDetails(BaseLiteLLMOpenAIResponseObject):
ttl_30m_tokens: int
ttl_12h_tokens: int
model_config = ConfigDict(extra="allow", frozen=True)
def _is_none(value: object) -> bool:
return value is None
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
audio_tokens: int | None = None
cached_tokens: int = 0
cached_tokens_details: CachedTokensDetails | None = None
cache_write_tokens: int | None = Field(default=None, exclude_if=_is_none)
cache_write_tokens_details: CacheWriteTokensDetails | None = Field(default=None, exclude_if=_is_none)
image_tokens: int | None = None
text_tokens: int | None = None
video_tokens: int | None = None

View file

@ -322,11 +322,19 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_creation_input_token_cost_above_272k_tokens_flex: float | None
cache_creation_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
cache_creation_input_token_cost_above_1hr: float | None
cache_creation_input_token_cost_above_12hr: ReadOnly[float | None]
cache_creation_input_token_cost_above_12hr_flex: ReadOnly[float | None]
cache_creation_input_token_cost_above_12hr_priority: ReadOnly[float | None]
cache_creation_input_token_cost_above_12hr_above_272k_tokens: ReadOnly[float | None]
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: ReadOnly[float | None]
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: ReadOnly[float | None]
cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_read_input_token_cost: float | None
cache_read_input_audio_token_cost: ReadOnly[float | None]
cache_read_input_audio_token_cost_flex: ReadOnly[float | None]
cache_read_input_audio_token_cost_priority: ReadOnly[float | None]
cache_read_input_image_token_cost: ReadOnly[float | None]
explicit_cache_read_input_token_cost: ReadOnly[float | None]
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
@ -342,6 +350,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_read_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
cache_read_input_token_cost_above_512k_tokens: float | None
cache_read_input_token_cost_batches: ReadOnly[float | None]
cache_read_input_audio_token_cost_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None]
@ -355,6 +364,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates
input_cost_per_character: float | None # only for vertex ai models
input_cost_per_audio_token: float | None
input_cost_per_audio_token_flex: ReadOnly[float | None]
input_cost_per_audio_token_priority: ReadOnly[float | None]
input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models
input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models
input_cost_per_token_above_100k_tokens: ReadOnly[float | None]
@ -1804,7 +1815,9 @@ class CompletionTokensDetailsWrapper(CompletionTokensDetails): # wrapper for ol
class CacheCreationTokenDetails(LiteLLMBaseModel):
ephemeral_5m_input_tokens: int | None = None
ephemeral_30m_input_tokens: int | None = None
ephemeral_1h_input_tokens: int | None = None
ephemeral_12h_input_tokens: int | None = None
class PromptTokensDetailsWrapper(
@ -3954,6 +3967,12 @@ class MirroredPricingParams(LiteLLMBaseModel):
explicit_cache_read_input_token_cost: float | None = None
cache_creation_input_token_cost: float | None = None
cache_creation_input_token_cost_above_1hr: float | None = None
cache_creation_input_token_cost_above_12hr: float | None = None
cache_creation_input_token_cost_above_12hr_flex: float | None = None
cache_creation_input_token_cost_above_12hr_priority: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None
tiered_pricing: list[dict[str, Any]] | None = None
@ -4006,14 +4025,19 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
cache_read_input_token_cost_above_100k_tokens_batches: float | None = None
cache_read_input_token_cost_above_200k_tokens_batches: float | None = None
cache_read_input_token_cost_above_272k_tokens_batches: float | None = None
cache_read_input_audio_token_cost_batches: float | None = None
cache_creation_input_token_cost_batches: float | None = None
cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None
cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None
cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None
cache_read_input_audio_token_cost: float | None = None
cache_read_input_audio_token_cost_flex: float | None = None
cache_read_input_audio_token_cost_priority: float | None = None
cache_read_input_image_token_cost: float | None = None
input_cost_per_character_above_128k_tokens: float | None = None
input_cost_per_audio_token: float | None = None
input_cost_per_audio_token_flex: float | None = None
input_cost_per_audio_token_priority: float | None = None
input_cost_per_token_cache_hit: float | None = None
input_cost_per_token_above_128k_tokens: float | None = None
input_cost_per_token_above_100k_tokens: float | None = None

View file

@ -6773,6 +6773,13 @@ def get_model_info_helper(
),
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
cache_read_input_audio_token_cost_flex=_model_info.get("cache_read_input_audio_token_cost_flex", None),
cache_read_input_audio_token_cost_priority=_model_info.get(
"cache_read_input_audio_token_cost_priority", None
),
cache_read_input_audio_token_cost_batches=_model_info.get(
"cache_read_input_audio_token_cost_batches", None
),
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
@ -6817,6 +6824,24 @@ def get_model_info_helper(
cache_creation_input_token_cost_above_1hr=_model_info.get(
"cache_creation_input_token_cost_above_1hr", None
),
cache_creation_input_token_cost_above_12hr=_model_info.get(
"cache_creation_input_token_cost_above_12hr", None
),
cache_creation_input_token_cost_above_12hr_flex=_model_info.get(
"cache_creation_input_token_cost_above_12hr_flex", None
),
cache_creation_input_token_cost_above_12hr_priority=_model_info.get(
"cache_creation_input_token_cost_above_12hr_priority", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", None
),
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority=_model_info.get(
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", None
),
off_peak_pricing=_model_info.get("off_peak_pricing", None),
input_cost_per_character=_model_info.get("input_cost_per_character", None),
input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None),
@ -6840,6 +6865,8 @@ def get_model_info_helper(
input_cost_per_second=_model_info.get("input_cost_per_second", None),
input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None),
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None),
input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None),
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None),

View file

@ -31137,11 +31137,15 @@
},
"gemini/gemini-2.5-flash": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 3e-08,
"cache_read_input_token_cost_batches": 3e-08,
"cache_read_input_token_cost_flex": 3e-08,
"cache_read_input_token_cost_priority": 5.4e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 3e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31466,11 +31470,15 @@
},
"gemini/gemini-2.5-flash-lite": {
"cache_read_input_audio_token_cost": 3e-08,
"cache_read_input_audio_token_cost_batches": 3e-08,
"cache_read_input_audio_token_cost_flex": 3e-08,
"cache_read_input_audio_token_cost_priority": 5.4e-08,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_batches": 1e-08,
"cache_read_input_token_cost_flex": 1e-08,
"cache_read_input_token_cost_priority": 1.8e-08,
"input_cost_per_audio_token": 3e-07,
"input_cost_per_audio_token_flex": 1.5e-07,
"input_cost_per_token": 1e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -31760,12 +31768,16 @@
},
"gemini/gemini-3.1-flash-lite": {
"cache_read_input_audio_token_cost": 5e-08,
"cache_read_input_audio_token_cost_batches": 2.5e-08,
"cache_read_input_audio_token_cost_flex": 2.5e-08,
"cache_read_input_audio_token_cost_priority": 9e-08,
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_batches": 1.25e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"deprecation_date": "2027-05-07",
"input_cost_per_audio_token": 5e-07,
"input_cost_per_audio_token_flex": 2.5e-07,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"input_cost_per_token_flex": 1.25e-07,
@ -31886,10 +31898,14 @@
},
"gemini/gemini-3-flash-preview": {
"cache_read_input_audio_token_cost": 1e-07,
"cache_read_input_audio_token_cost_batches": 1e-07,
"cache_read_input_audio_token_cost_flex": 1e-07,
"cache_read_input_audio_token_cost_priority": 1.8e-07,
"cache_read_input_token_cost": 5e-08,
"cache_read_input_token_cost_batches": 5e-08,
"cache_read_input_token_cost_flex": 5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_audio_token_flex": 5e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "gemini",
"max_input_tokens": 1048576,
@ -36795,6 +36811,12 @@
},
"gpt-6-astra": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_12hr": 2.5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001,
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-05,
"cache_creation_input_token_cost_above_12hr_priority": 5e-05,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
@ -36880,6 +36902,12 @@
},
"gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -36957,6 +36985,12 @@
},
"gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_12hr": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06,
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-07,
"cache_creation_input_token_cost_above_12hr_priority": 5e-07,
"cache_creation_input_token_cost_decisions": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
@ -37043,6 +37077,12 @@
},
"gpt-5.6": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_12hr": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37117,6 +37157,12 @@
},
"gpt-5.6-sol": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_12hr": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
@ -37193,6 +37239,12 @@
},
"gpt-5.6-terra": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
@ -37268,6 +37320,12 @@
},
"gpt-5.6-luna": {
"cache_creation_input_token_cost": 2.5e-07,
"cache_creation_input_token_cost_above_12hr": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-07,
"cache_creation_input_token_cost_above_12hr_priority": 1e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
@ -45847,6 +45905,7 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -45872,6 +45931,7 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -49587,6 +49647,7 @@
"supports_vision": false
},
"us-gov.openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -49606,6 +49667,7 @@
"supports_tool_choice": true
},
"us-gov.openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -60156,6 +60218,7 @@
}
},
"bedrock_mantle/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7.5e-08,
"input_cost_per_token_flex": 7.5e-08,
@ -60181,6 +60244,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3.5e-08,
"input_cost_per_token_flex": 3.5e-08,
@ -60206,6 +60270,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7e-08,
"input_cost_per_token_flex": 7e-08,
@ -60229,6 +60294,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
@ -68080,6 +68146,7 @@
"supports_vision": false
},
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68099,6 +68166,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68377,6 +68445,7 @@
"supports_vision": false
},
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68396,6 +68465,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68784,6 +68854,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68803,6 +68874,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -68943,6 +69015,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68962,6 +69035,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -74917,6 +74991,12 @@
},
"gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,

View file

@ -6985,6 +6985,106 @@ GPT_5_6_RESPONSES_CACHE_READ: Final = CostTrackingTestCase(
),
)
GPT_5_6_RESPONSES_CACHE_WRITE_12H: Final = CostTrackingTestCase(
name="gpt-5.6-responses_cache_write_12h",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-5.6",
endpoint="/v1/responses",
request={
"model": "$MODEL",
"input": "summarize this text",
"prompt_cache_options": {"mode": "implicit", "ttl": "12h"},
},
response=JsonResponse(
content_type="application/json",
body={
"id": "resp_$REQUEST_ID",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "gpt-5.6",
"output": [
{
"type": "message",
"id": "msg_$REQUEST_ID",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "scripted response", "annotations": []}],
}
],
"usage": {
"input_tokens": 12928,
"output_tokens": 380,
"total_tokens": 13308,
"input_tokens_details": {
"cached_tokens": 0,
"cache_write_tokens": 5000,
"cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000},
},
"output_tokens_details": {"reasoning_tokens": 0},
},
},
),
expected=ExactExpected(
spend=0.0344865,
input_cost=0.0291865,
output_cost=0.00532,
prompt_tokens=12928,
completion_tokens=380,
cache_creation_cost=0.0153125,
),
)
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE: Final = CostTrackingTestCase(
name="gpt-6-luna-responses_cache_write_12h_no_12h_rate",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="gpt-6-luna",
endpoint="/v1/responses",
request={
"model": "$MODEL",
"input": "summarize this text",
"prompt_cache_options": {"mode": "implicit", "ttl": "12h"},
},
response=JsonResponse(
content_type="application/json",
body={
"id": "resp_$REQUEST_ID",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "gpt-6-luna",
"output": [
{
"type": "message",
"id": "msg_$REQUEST_ID",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "scripted response", "annotations": []}],
}
],
"usage": {
"input_tokens": 12928,
"output_tokens": 380,
"total_tokens": 13308,
"input_tokens_details": {
"cached_tokens": 0,
"cache_write_tokens": 5000,
"cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000},
},
"output_tokens_details": {"reasoning_tokens": 0},
},
},
),
expected=ExactExpected(
spend=0.0016078,
input_cost=0.0014178,
output_cost=0.00019,
prompt_tokens=12928,
completion_tokens=380,
cache_creation_cost=0.000625,
),
)
GPT_5_6_RESPONSES_REASONING: Final = CostTrackingTestCase(
name="gpt-5.6-responses_reasoning",
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
@ -11913,6 +12013,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
OMNI_MODERATIONS_NEXT_SINGLE,
OMNI_MODERATIONS_NEXT_LIST,
GPT_5_6_RESPONSES_CACHE_READ,
GPT_5_6_RESPONSES_CACHE_WRITE_12H,
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
GPT_5_6_RESPONSES_REASONING,
GPT_5_6_RESPONSES_STREAM,
GPT_5_6_RESPONSES_STREAM_CACHE_READ,

View file

@ -174,6 +174,8 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
},
},
"gpt-5.6": {
"cache_creation_input_token_cost": 2.1875e-06,
"cache_creation_input_token_cost_above_12hr": 4.375e-06,
"cache_read_input_token_cost": 1.75e-07,
"input_cost_per_audio_token": 4e-05,
"input_cost_per_token": 1.75e-06,

View file

@ -85,6 +85,12 @@ class CostMapEntry(BaseModel):
cache_read_input_token_cost: float | None = None
cache_creation_input_token_cost: float | None = None
cache_creation_input_token_cost_above_1hr: float | None = None
cache_creation_input_token_cost_above_12hr: float | None = None
cache_creation_input_token_cost_above_12hr_flex: float | None = None
cache_creation_input_token_cost_above_12hr_priority: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None
cache_creation_input_token_cost_above_1hr_above_200k_tokens: float | None = None
cache_read_input_token_cost_above_200k_tokens: float | None = None
cache_creation_input_token_cost_above_200k_tokens: float | None = None

View file

@ -65,8 +65,8 @@ def test_missing_cache_read_rate_resolves_to_input_rate(prompt_tokens, read_rate
prompt_cost, _ = generic_cost_per_token(
"policy-fixture", usage, "openai", service_tier=service_tier, model_info=info
)
assert billed[4] == pytest.approx(read_rate if read_rate is not None else billed[0])
assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4])
assert billed.cache_read_rate == pytest.approx(read_rate if read_rate is not None else billed.input_rate)
assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed.input_rate + 100 * billed.cache_read_rate)
def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None:
@ -248,7 +248,7 @@ def test_missing_cache_read_uses_off_peak_input_rate():
}
when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc)
billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when)
assert billed[0] == billed[4] == 5e-6
assert billed.input_rate == billed.cache_read_rate == 5e-6
def test_reasoning_tokens_no_price_set(_local_model_cost_map):
@ -324,8 +324,6 @@ def test_reasoning_tokens_gemini(_local_model_cost_map):
)
def test_image_tokens_with_custom_pricing():
"""Test that image_tokens in completion are properly costed with output_cost_per_image_token."""
from unittest.mock import patch
@ -578,7 +576,7 @@ def test_get_token_base_cost_picks_highest_crossed_tier():
}
usage = Usage(prompt_tokens=150_000, completion_tokens=10, total_tokens=150_010)
prompt_base_cost = _get_token_base_cost(model_info, usage)[0]
prompt_base_cost = _get_token_base_cost(model_info, usage).input_rate
assert prompt_base_cost == 9e-6
@ -779,12 +777,12 @@ def test_get_token_base_cost_weekend_only_off_peak_rate():
saturday_peak_hours = _get_token_base_cost(
model_info, usage, current_time=datetime(2026, 8, 29, 2, 0, tzinfo=timezone.utc)
)
assert saturday_peak_hours[:2] == (5e-7, 1e-6)
assert (saturday_peak_hours.input_rate, saturday_peak_hours.output_rate) == (5e-7, 1e-6)
monday_same_hours = _get_token_base_cost(
model_info, usage, current_time=datetime(2026, 8, 24, 2, 0, tzinfo=timezone.utc)
)
assert monday_same_hours[:2] == (1e-6, 2e-6)
assert (monday_same_hours.input_rate, monday_same_hours.output_rate) == (1e-6, 2e-6)
def test_get_token_base_cost_applies_off_peak_pricing():
@ -810,14 +808,14 @@ def test_get_token_base_cost_applies_off_peak_pricing():
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc))
assert off_peak[0] == 5e-7
assert off_peak[1] == 1e-6
assert off_peak[4] == 5e-8
assert off_peak.input_rate == 5e-7
assert off_peak.output_rate == 1e-6
assert off_peak.cache_read_rate == 5e-8
peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc))
assert peak[0] == 1e-6
assert peak[1] == 2e-6
assert peak[4] == 1e-7
assert peak.input_rate == 1e-6
assert peak.output_rate == 2e-6
assert peak.cache_read_rate == 1e-7
def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates():
@ -843,8 +841,8 @@ def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates():
},
)
result = _get_token_base_cost(model_info, usage, current_time=when)
assert result[0] == 1e-6
assert result[1] == 2e-6
assert result.input_rate == 1e-6
assert result.output_rate == 2e-6
def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset():
@ -864,8 +862,8 @@ def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset():
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
result = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc))
assert result[0] == 5e-7
assert result[1] == 2e-6
assert result.input_rate == 5e-7
assert result.output_rate == 2e-6
def test_get_token_base_cost_off_peak_wins_over_threshold():
@ -891,12 +889,12 @@ def test_get_token_base_cost_off_peak_wins_over_threshold():
usage = Usage(prompt_tokens=250000, completion_tokens=250000, total_tokens=500000)
off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc))
assert off_peak[0] == 5e-7
assert off_peak[1] == 1e-6
assert off_peak.input_rate == 5e-7
assert off_peak.output_rate == 1e-6
peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc))
assert peak[0] == 3e-6
assert peak[1] == 4e-6
assert peak.input_rate == 3e-6
assert peak.output_rate == 4e-6
def test_get_model_info_propagates_off_peak_fields():
@ -948,10 +946,10 @@ def test_get_token_base_cost_off_peak_wins_over_tiered_pricing():
usage = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100)
inside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc))
assert inside[:2] == (5e-7, 1e-6)
assert (inside.input_rate, inside.output_rate) == (5e-7, 1e-6)
outside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc))
assert outside[:2] == (3e-6, 6e-6)
assert (outside.input_rate, outside.output_rate) == (3e-6, 6e-6)
def _register_off_peak_reasoning_model(
@ -1153,17 +1151,17 @@ def test_get_token_base_cost_off_peak_cache_creation_rate():
inside_window = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)
inside = _get_token_base_cost(model_info, usage, current_time=inside_window)
assert inside[2] == 5e-7
assert inside[3] == 2e-6
assert inside.cache_creation_rate == 5e-7
assert inside.cache_creation_rates_by_ttl["1hr"] == 2e-6
outside = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc))
assert outside[2] == 1.25e-6
assert outside.cache_creation_rate == 1.25e-6
without_key = cast(
ModelInfo,
{**model_info, "off_peak_pricing": {"hours_utc": "16:30-00:30", "input_cost_per_token": 5e-7}},
)
assert _get_token_base_cost(without_key, usage, current_time=inside_window)[2] == 1.25e-6
assert _get_token_base_cost(without_key, usage, current_time=inside_window).cache_creation_rate == 1.25e-6
def test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_creation_rates():
@ -1583,19 +1581,14 @@ def test_generic_cost_per_token_tier_1hr_cache_rate_preserves_zero_and_fallback(
),
)
costs: Final = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openrouter")
breakdown: Final = get_token_type_cost_breakdown(
model=model, usage=usage, custom_llm_provider="openrouter"
)
breakdown: Final = get_token_type_cost_breakdown(model=model, usage=usage, custom_llm_provider="openrouter")
ordinary_rate: Final = cache_rates.get("cache_creation_input_token_cost")
expected_write_cost: Final = (
(20 - one_hour_tokens) * (ordinary_rate if ordinary_rate is not None else 1.0)
+ one_hour_tokens * expected_write_rate
)
expected_write_cost: Final = (20 - one_hour_tokens) * (
ordinary_rate if ordinary_rate is not None else 1.0
) + one_hour_tokens * expected_write_rate
assert costs == pytest.approx((40 * 1.0 + 40 * 0.1 + expected_write_cost, 10 * 2.0))
assert breakdown.cache_creation_cost == pytest.approx(expected_write_cost)
assert breakdown.rates is not None
assert breakdown.rates.cache_creation_input_token_cost_above_1hr == expected_write_rate
finally:
litellm.model_cost.pop(model, None)
@ -2020,8 +2013,8 @@ def test_calculate_cache_writing_cost():
result = calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr},
)
# Expected calculation:
@ -2041,8 +2034,8 @@ def test_calculate_cache_writing_cost():
result_no_details = calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens_no_details,
cache_creation_token_details=cache_creation_token_details_none,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=cache_creation_cost_fallback,
cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr},
)
# Expected calculation when no details: 1000 * 5e-06 = 0.005
@ -2059,15 +2052,13 @@ def test_calculate_cache_writing_cost():
result_partial = calculate_cache_writing_cost(
cache_creation_tokens=500,
cache_creation_token_details=cache_creation_token_details_partial,
cache_creation_cost_above_1hr=6e-06,
cache_creation_cost=3e-06,
cache_creation_cost_by_ttl={"1hr": 6e-06, "12hr": 6e-06},
)
# Expected calculation: 0 (for None 5m tokens) + (100 * 6e-06) = 0.0006
expected_cost_partial = (0.0) + (100 * 6e-06)
expected_cost_partial = (400 * 3e-06) + (100 * 6e-06)
assert round(result_partial, 6) == round(expected_cost_partial, 6)
assert round(result_partial, 6) == 0.0006
# Test case 4: Zero costs
result_zero = calculate_cache_writing_cost(
@ -2075,13 +2066,119 @@ def test_calculate_cache_writing_cost():
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_5m_input_tokens=50, ephemeral_1h_input_tokens=950
),
cache_creation_cost_above_1hr=0.0,
cache_creation_cost=0.0,
cache_creation_cost_by_ttl={"1hr": 0.0, "12hr": 0.0},
)
assert result_zero == 0.0
@pytest.mark.parametrize("field", CacheCreationTokenDetails.model_fields)
def test_calculate_cache_writing_cost_bills_every_reported_lifetime(field: str) -> None:
cost = calculate_cache_writing_cost(
cache_creation_tokens=10,
cache_creation_token_details=CacheCreationTokenDetails(**{field: 10}),
cache_creation_cost=2.0,
cache_creation_cost_by_ttl={"1hr": 2.0, "12hr": 2.0},
)
assert cost == 20.0
def test_calculate_cache_writing_cost_bills_writes_of_an_unknown_lifetime_at_the_base_rate() -> None:
cost = calculate_cache_writing_cost(
cache_creation_tokens=10,
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_30m_input_tokens=2, ephemeral_12h_input_tokens=3
),
cache_creation_cost=1.0,
cache_creation_cost_by_ttl={"1hr": 4.0, "12hr": 4.0},
)
assert cost == 2 * 1.0 + 3 * 4.0 + 5 * 1.0
@pytest.mark.parametrize(
("prompt_tokens", "service_tier", "write_rate", "twelve_hour_rate"),
[
(10_000, "default", 2e-6, 4e-6),
(10_000, "flex", 1e-6, 2e-6),
(10_000, "priority", 3e-6, 6e-6),
(300_000, "default", 4e-6, 8e-6),
(300_000, "flex", 2e-6, 4e-6),
(300_000, "priority", 6e-6, 12e-6),
],
)
def test_generic_cost_per_token_prices_12h_cache_writes_at_prompt_threshold(
prompt_tokens: int, service_tier: str, write_rate: float, twelve_hour_rate: float
) -> None:
model_info: Final[ModelInfo] = {
"input_cost_per_token": 1e-6,
"output_cost_per_token": 2e-6,
"cache_creation_input_token_cost": 2e-6,
"cache_creation_input_token_cost_flex": 1e-6,
"cache_creation_input_token_cost_priority": 3e-6,
"cache_creation_input_token_cost_above_272k_tokens": 4e-6,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2e-6,
"cache_creation_input_token_cost_above_272k_tokens_priority": 6e-6,
"cache_creation_input_token_cost_above_12hr": 4e-6,
"cache_creation_input_token_cost_above_12hr_flex": 2e-6,
"cache_creation_input_token_cost_above_12hr_priority": 6e-6,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 8e-6,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 4e-6,
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 12e-6,
}
usage: Final = Usage(
prompt_tokens=prompt_tokens,
completion_tokens=0,
total_tokens=prompt_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
cache_creation_tokens=5_000,
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_30m_input_tokens=3_000, ephemeral_12h_input_tokens=2_000
),
),
)
result: Final = generic_cost_per_token(
model="test-12h-cache-write",
usage=usage,
custom_llm_provider="openai",
service_tier=None if service_tier == "default" else service_tier,
model_info=model_info,
)
assert result[0] == pytest.approx(
(usage.prompt_tokens - usage.prompt_tokens_details.cache_creation_tokens) * model_info["input_cost_per_token"]
+ 3_000 * write_rate
+ 2_000 * twelve_hour_rate
)
def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> None:
model_info: Final[ModelInfo] = {
"input_cost_per_token": 1e-6,
"output_cost_per_token": 2e-6,
"cache_creation_input_token_cost": 3e-6,
}
usage: Final = Usage(
prompt_tokens=100,
completion_tokens=0,
total_tokens=100,
prompt_tokens_details=PromptTokensDetailsWrapper(
cache_creation_tokens=100,
cache_creation_token_details=CacheCreationTokenDetails(ephemeral_12h_input_tokens=100),
),
)
result: Final = generic_cost_per_token(
model="test-12h-cache-write-fallback",
usage=usage,
custom_llm_provider="openai",
model_info=model_info,
)
assert result[0] == pytest.approx(100 * model_info["cache_creation_input_token_cost"])
def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
"""
Regression test: when cache_creation_tokens is 0 but cache_creation_token_details
@ -2119,7 +2216,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
prompt_base_cost=0.0,
cache_read_cost=0.0,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr},
)
# Expected: (100 * 3.75e-06) + (200 * 6e-06) = 0.000375 + 0.0012 = 0.001575
@ -2330,10 +2427,6 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(_local_mo
assert round(cost, 10) == round(expected_cost, 10)
def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map):
usage = Usage(
prompt_tokens=0,
@ -2512,8 +2605,6 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode
assert base == located
def test_vertex_uplift_invalid_multiplier_defaults_to_one():
"""A malformed multiplier in the cost map degrades to base pricing, never raises."""
from litellm.litellm_core_utils.llm_cost_calc.utils import (
@ -2569,10 +2660,10 @@ def test_flat_input_price_keeps_other_token_thresholds(field: str) -> None:
model_info=info,
usage=Usage(prompt_tokens=300_001, completion_tokens=3, total_tokens=300_004),
)
assert rates[0] == 0.001
assert rates[1] == (0.005 if field == "output_cost_per_token" else 0.002)
assert rates[2] == (0.005 if field == "cache_creation_input_token_cost" else 0.004)
assert rates[4] == (0.005 if field == "cache_read_input_token_cost" else 0.003)
assert rates.input_rate == 0.001
assert rates.output_rate == (0.005 if field == "output_cost_per_token" else 0.002)
assert rates.cache_creation_rate == (0.005 if field == "cache_creation_input_token_cost" else 0.004)
assert rates.cache_read_rate == (0.005 if field == "cache_read_input_token_cost" else 0.003)
@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001, 512_000, 512_001))
@ -2599,10 +2690,12 @@ def test_mixed_token_thresholds_preserve_each_crossed_rate(prompt_tokens: int, h
model_info=info,
usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=3, total_tokens=prompt_tokens + 3),
)
assert rates[0] == (0.005 if prompt_tokens > 200_000 else 0.001)
assert rates[1] == (higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002)
assert rates[2] == (0.007 if prompt_tokens > 200_000 else 0.004)
assert rates[4] == (higher_rate if prompt_tokens > 512_000 else 0.003)
assert rates.input_rate == (0.005 if prompt_tokens > 200_000 else 0.001)
assert rates.output_rate == (
higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002
)
assert rates.cache_creation_rate == (0.007 if prompt_tokens > 200_000 else 0.004)
assert rates.cache_read_rate == (higher_rate if prompt_tokens > 512_000 else 0.003)
def test_threshold_keys_exclude_service_tier_variants():
@ -2626,8 +2719,7 @@ def test_threshold_keys_exclude_service_tier_variants():
},
)
usage = Usage(prompt_tokens=350_000, completion_tokens=1_000, total_tokens=351_000)
prompt_base, *_ = _get_token_base_cost(model_info=model_info, usage=usage)
assert prompt_base == 3e-6
assert _get_token_base_cost(model_info=model_info, usage=usage).input_rate == 3e-6
@pytest.mark.parametrize(
@ -2897,7 +2989,7 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp
cache_read_input_token_cost=6e-7,
cache_read_input_audio_token_cost=6e-7,
cache_creation_input_token_cost=7.5e-6,
cache_creation_input_token_cost_above_1hr=7.5e-6,
cache_creation_input_token_cost_by_ttl={"1hr": 7.5e-6, "12hr": 7.5e-6},
output_cost_per_reasoning_token=3e-5,
)
assert breakdown.cache_read_cost == pytest.approx(200_000 * rates.cache_read_input_token_cost)
@ -3713,6 +3805,23 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a
3e-7,
id="no-1h-price-uses-the-off-peak-write-price",
),
pytest.param(
{
"tiered_pricing": [
{
"range": [0, 128000],
"input_cost_per_token": 2e-7,
"output_cost_per_token": 1.25e-6,
"cache_creation_input_token_cost": 2.5e-7,
}
],
"off_peak_pricing": {"hours_utc": "00:00-23:59", "cache_creation_input_token_cost": 3e-7},
},
datetime(2026, 9, 14, 12, tzinfo=timezone.utc),
3e-7,
3e-7,
id="tiered-no-1h-price-uses-the-off-peak-write-price",
),
),
)
def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_path(
@ -3724,10 +3833,17 @@ def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_
model_info = {"input_cost_per_token": 2e-7, "output_cost_per_token": 1.25e-6, **cache_rates}
usage = Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11)
_, _, creation, creation_1h, _ = _get_token_base_cost(model_info, usage, current_time=current_time)
rates = _get_token_base_cost(model_info, usage, current_time=current_time)
assert creation == pytest.approx(expected_creation)
assert creation_1h == pytest.approx(expected_creation_1h)
assert rates.cache_creation_rate == pytest.approx(expected_creation)
assert calculate_cache_writing_cost(
cache_creation_tokens=3,
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_1h_input_tokens=1, ephemeral_12h_input_tokens=2
),
cache_creation_cost=rates.cache_creation_rate,
cache_creation_cost_by_ttl=rates.cache_creation_rates_by_ttl,
) == pytest.approx(3 * expected_creation_1h)
def _image_response(num_images: int = 1, usage: ImageUsage | None = None) -> ImageResponse:
@ -3787,8 +3903,6 @@ def test_route_image_generation_cost_openai_honors_deployment_input_cost_per_ima
assert cost == pytest.approx(0.07)
@pytest.mark.parametrize(
("custom_llm_provider", "model"),
[
@ -4206,17 +4320,28 @@ def test_token_breakdown_uses_supplied_pricing_without_catalog_lookup(rate: floa
"cache_creation_input_token_cost": rate,
}
usage: Final = Usage(
prompt_tokens=20, completion_tokens=3, total_tokens=23,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), cache_creation_input_tokens=5,
prompt_tokens=20,
completion_tokens=3,
total_tokens=23,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5),
cache_creation_input_tokens=5,
)
prompt_cost, output_cost = generic_cost_per_token(
model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", model_info=model_info,
model="supplied-pricing-test",
usage=usage,
custom_llm_provider="bedrock",
model_info=model_info,
)
breakdown: Final = get_token_type_cost_breakdown(
model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", model_info=model_info,
model="supplied-pricing-test",
usage=usage,
custom_llm_provider="bedrock",
model_info=model_info,
)
assert breakdown.cache_read_cost == pytest.approx(5 * rate)
assert breakdown.cache_creation_cost == pytest.approx(5 * rate)
assert breakdown.rates is not None
assert prompt_cost == pytest.approx(10 * breakdown.rates.input_cost_per_token + breakdown.cache_read_cost + breakdown.cache_creation_cost)
assert prompt_cost == pytest.approx(
10 * breakdown.rates.input_cost_per_token + breakdown.cache_read_cost + breakdown.cache_creation_cost
)
assert output_cost == pytest.approx(3 * breakdown.rates.output_cost_per_token)

View file

@ -11,6 +11,7 @@ import respx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
TIER_MODEL: Final = "tier-priced-test-model"
@ -31,6 +32,20 @@ PROMPT_TOKENS: Final = 1000
CACHED_TOKENS: Final = 200
COMPLETION_TOKENS: Final = 500
TIER_API_BASE: Final = "https://tier-pricing.invalid/v1"
AUDIO_TIER_MODEL: Final = "tier-priced-gemini-audio-test-model"
AUDIO_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
{
"input_cost_per_token": 4e-06,
"output_cost_per_token": 8e-06,
"cache_read_input_token_cost": 1e-06,
"input_cost_per_audio_token": 3e-06,
"cache_read_input_audio_token_cost": 2e-06,
"input_cost_per_audio_token_flex": 1e-06,
"input_cost_per_audio_token_priority": 2e-06,
"cache_read_input_audio_token_cost_flex": 4e-07,
"cache_read_input_audio_token_cost_priority": 8e-07,
}
)
def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float:
@ -78,6 +93,53 @@ def test_completion_cost_bills_the_price_columns_of_the_service_tier(
assert cost == pytest.approx(_cost_at(TIER_ROW, column_suffix))
@pytest.mark.parametrize(
("service_tier", "audio_rate_key", "cache_audio_rate_key"),
(
pytest.param("flex", "input_cost_per_audio_token_flex", "cache_read_input_audio_token_cost_flex"),
pytest.param("priority", "input_cost_per_audio_token_priority", "cache_read_input_audio_token_cost_priority"),
),
)
def test_gemini_audio_and_cached_audio_use_service_tier_rates(
local_model_cost_map: None, service_tier: str, audio_rate_key: str, cache_audio_rate_key: str
) -> None:
litellm.register_model(
{
AUDIO_TIER_MODEL: {
"litellm_provider": "gemini",
"mode": "chat",
**AUDIO_TIER_ROW,
}
}
)
usage: Final = Usage(
prompt_tokens=40,
completion_tokens=0,
total_tokens=40,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=20,
audio_tokens=20,
cached_tokens=10,
cached_tokens_details=CachedTokensDetails(audio_tokens=10),
),
)
response: Final = ModelResponse(model=AUDIO_TIER_MODEL, usage=usage)
cost: Final = litellm.completion_cost(
completion_response=response,
model=AUDIO_TIER_MODEL,
custom_llm_provider="gemini",
service_tier=service_tier,
)
expected: Final = (
20 * AUDIO_TIER_ROW["input_cost_per_token"]
+ 10 * AUDIO_TIER_ROW[audio_rate_key]
+ 10 * AUDIO_TIER_ROW[cache_audio_rate_key]
)
assert cost == pytest.approx(expected)
LONG_CONTEXT_TIER_MODEL: Final = "long-context-tier-priced-test-model"
LONG_CONTEXT_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
{

View file

@ -9,6 +9,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import (
used_pricing_keys,
zero_cost_warning,
)
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.utils import CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, Usage
PER_SECOND_ENTRY: Final = {"cost_per_second": 0.00042}
@ -135,6 +136,36 @@ def test_audio_tokens_need_the_audio_rates() -> None:
assert diagnostic["missing_pricing_keys"] == ("input_cost_per_audio_token", "output_cost_per_audio_token")
def test_cached_audio_uses_its_cache_rate_in_zero_cost_diagnostic() -> None:
usage: Final = Usage(
prompt_tokens=20,
completion_tokens=0,
total_tokens=20,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=0,
audio_tokens=20,
cached_tokens=20,
cached_tokens_details=CachedTokensDetails(audio_tokens=20),
),
)
pricing_entry: Final = {
"input_cost_per_token": 1e-6,
"input_cost_per_audio_token": 2e-6,
"cache_read_input_audio_token_cost": 0,
}
assert used_pricing_keys(usage) == ("cache_read_input_audio_token_cost",)
assert (
diagnose_zero_cost(
usage=usage,
pricing_model="gemini-audio",
pricing_entry=pricing_entry,
calculation_failed=False,
)
is None
)
def test_warning_names_the_request_the_entry_the_missing_keys_and_the_counter() -> None:
diagnostic = diagnose_zero_cost(
usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False

View file

@ -16,8 +16,10 @@ from pydantic import BaseModel
import litellm
from litellm import ModelResponse, completion, embedding, image_generation
from litellm._logging import verbose_logger
from litellm.cost_calculator import batch_cost_calculator
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details
from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
@ -30,8 +32,9 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
normalize_gemini_speech_config,
reject_unknown_gemini_tts_voice_keys,
)
from litellm.types.llms.base import CachedTokensDetails
from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata
from litellm.types.utils import ChoiceLogprobs, Usage
from litellm.types.utils import ChoiceLogprobs, ModelInfo, Usage
from litellm.utils import _invalidate_model_cost_lowercase_map, CustomStreamWrapper
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.integrations.custom_logger import CustomLogger
@ -1515,6 +1518,7 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities():
v = VertexGeminiConfig()
usage_metadata = {
"promptTokenCount": 210,
"cachedContentTokenCount": 54,
"candidatesTokenCount": 50,
"totalTokenCount": 260,
"promptTokensDetails": [
@ -1536,9 +1540,10 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities():
usage_metadata = UsageMetadata(**usage_metadata)
result = v.calculate_usage(completion_response={"usageMetadata": usage_metadata})
# prompt details are total - cached per modality
assert result.prompt_tokens_details.text_tokens == 16 # 20 - 4
assert result.prompt_tokens_details.image_tokens == 140 # (90 + 100) - (40 + 10)
assert result.prompt_tokens_details.text_tokens == 20
assert result.prompt_tokens_details.image_tokens == 190
assert result.prompt_tokens_details.cached_tokens == 54
assert result.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=4, image_tokens=50)
# candidates details accumulate duplicate modalities
assert result.completion_tokens_details.text_tokens == 20 # 15 + 5
@ -1546,6 +1551,118 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities():
assert result.completion_tokens == 50
def _gemini_usage_with_cached_audio(cached_audio_tokens: int) -> Usage:
cache_tokens_details: Final = (
(
{"modality": "TEXT", "tokenCount": 5},
{"modality": "AUDIO", "tokenCount": cached_audio_tokens},
)
if cached_audio_tokens > 0
else ({"modality": "TEXT", "tokenCount": 5},)
)
completion_response: Final = {
"usageMetadata": {
"promptTokenCount": 40,
"cachedContentTokenCount": 5 + cached_audio_tokens,
"promptTokensDetails": (
{"modality": "TEXT", "tokenCount": 20},
{"modality": "AUDIO", "tokenCount": 20},
),
"cacheTokensDetails": cache_tokens_details,
"candidatesTokenCount": 7,
"totalTokenCount": 47,
}
}
return VertexGeminiConfig.calculate_usage(completion_response=completion_response)
def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_batch_rate() -> None:
model_info: Final[ModelInfo] = {
"input_cost_per_token": 2e-6,
"input_cost_per_token_batches": 1e-6,
"input_cost_per_audio_token": 6e-6,
"input_cost_per_audio_token_batches": 3e-6,
"cache_read_input_token_cost": 8e-7,
"cache_read_input_token_cost_batches": 4e-7,
"cache_read_input_audio_token_cost": 5e-7,
"cache_read_input_audio_token_cost_batches": 2e-7,
}
usage: Final = _gemini_usage_with_cached_audio(5)
expected_prompt_cost: Final = (
15 * model_info["input_cost_per_token_batches"]
+ 15 * model_info["input_cost_per_audio_token_batches"]
+ 5 * model_info["cache_read_input_token_cost_batches"]
+ 5 * model_info["cache_read_input_audio_token_cost_batches"]
)
prompt_cost, _ = batch_cost_calculator(
usage=usage,
model="gemini-audio-batch-test-model",
custom_llm_provider="gemini",
model_info=model_info,
)
assert prompt_cost == pytest.approx(expected_prompt_cost)
def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_standard_rate() -> None:
model_info: Final[ModelInfo] = {
"input_cost_per_token": 2e-6,
"input_cost_per_audio_token": 6e-6,
"cache_read_input_token_cost": 8e-7,
"cache_read_input_audio_token_cost": 5e-7,
}
usage: Final = _gemini_usage_with_cached_audio(5)
expected_prompt_cost: Final = (
15 * model_info["input_cost_per_token"]
+ 15 * model_info["input_cost_per_audio_token"]
+ 5 * model_info["cache_read_input_token_cost"]
+ 5 * model_info["cache_read_input_audio_token_cost"]
)
prompt_cost, _ = generic_cost_per_token(
model="gemini-audio-standard-test-model",
usage=usage,
custom_llm_provider="gemini",
model_info=model_info,
)
assert prompt_cost == pytest.approx(expected_prompt_cost)
def test_gemini_usage_totals_are_subtracted_once_and_audio_cache_rate_matches_audio_rate() -> None:
model_info: Final[ModelInfo] = {
"input_cost_per_token": 2e-6,
"input_cost_per_audio_token": 6e-6,
"cache_read_input_token_cost": 8e-7,
"cache_read_input_audio_token_cost": 6e-6,
}
cached_usage: Final = _gemini_usage_with_cached_audio(5)
uncached_usage: Final = _gemini_usage_with_cached_audio(0)
cached_details: Final = parse_prompt_tokens_details(cached_usage)
cached_prompt_cost, _ = generic_cost_per_token(
model="gemini-audio-standard-test-model",
usage=cached_usage,
custom_llm_provider="gemini",
model_info=model_info,
)
uncached_prompt_cost, _ = generic_cost_per_token(
model="gemini-audio-standard-test-model",
usage=uncached_usage,
custom_llm_provider="gemini",
model_info=model_info,
)
assert cached_usage.prompt_tokens_details.text_tokens == 20
assert cached_usage.prompt_tokens_details.audio_tokens == 20
assert cached_usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(
text_tokens=5, audio_tokens=5
)
assert cached_details["text_tokens"] == 15
assert cached_details["audio_tokens"] == 15
assert cached_prompt_cost == pytest.approx(uncached_prompt_cost)
def test_vertex_ai_map_thinking_param_with_budget_tokens_0():
"""
If budget_tokens is 0, do not set includeThoughts to True
@ -6150,7 +6267,7 @@ def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt():
def test_vertex_ai_usage_metadata_with_document_tokens_cached():
"""Test that cached DOCUMENT tokens are correctly subtracted from prompt text tokens."""
"""Test cached DOCUMENT token details and unchanged prompt cost."""
v = VertexGeminiConfig()
usage_metadata_dict = {
@ -6174,10 +6291,24 @@ def test_vertex_ai_usage_metadata_with_document_tokens_cached():
completion_response = {"usageMetadata": usage_metadata_dict}
result = v.calculate_usage(completion_response=completion_response)
# DOCUMENT cached tokens map to cached_text_tokens, so:
# text_tokens = (8 TEXT + 774 DOCUMENT) - 400 cached = 382
assert result.prompt_tokens_details.text_tokens == 382, "text_tokens should be (8 + 774) - 400 cached = 382"
assert result.prompt_tokens_details.text_tokens == 782
assert result.prompt_tokens_details.cached_tokens == 400
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 400
model_info: Final[ModelInfo] = {
"input_cost_per_token": 2e-6,
"cache_read_input_token_cost": 5e-7,
}
prompt_cost, _ = generic_cost_per_token(
model="gemini-document-cache-test-model",
usage=result,
custom_llm_provider="gemini",
model_info=model_info,
)
assert prompt_cost == pytest.approx(
382 * model_info["input_cost_per_token"] + 400 * model_info["cache_read_input_token_cost"]
)
def test_async_streaming_uses_custom_client():

View file

@ -3398,7 +3398,9 @@ class TestUsageTransformation:
assert response_usage.input_tokens_details is not None
assert response_usage.input_tokens_details.cached_tokens == 5
assert response_usage.input_tokens_details.text_tokens == 8
assert "cache_write_tokens" not in response_usage.input_tokens_details.model_dump()
assert {"cache_write_tokens", "cache_write_tokens_details"}.isdisjoint(
response_usage.input_tokens_details.model_dump()
)
def test_transform_usage_with_cached_tokens_gemini(self):
"""Test that cached_tokens from Gemini are properly transformed to input_tokens_details"""

View file

@ -8,7 +8,8 @@ import asyncio
import copy
import json
import logging
from collections.abc import Awaitable, Callable
import math
from collections.abc import Awaitable, Callable, Mapping
from pathlib import Path
from importlib import import_module
from typing import Final
@ -17,10 +18,12 @@ from unittest.mock import AsyncMock, patch
import httpx
import pytest
import respx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.llms.openai import ResponsesAPIResponse
from tests.unit.llms.bedrock.slow_upstream import (
STREAM_TIMEOUT_SECONDS,
UPSTREAM_FIRST_BYTE_SECONDS,
@ -333,7 +336,9 @@ async def test_aresponses_sends_the_declared_reasoning_effort_level_rounded_up(
def _effort_levels_disabled_by(entry: dict) -> tuple[str, ...]:
return tuple(level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False)
return tuple(
level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False
)
def _gpt_5_row_disabling_a_level(provider: str) -> tuple[str, str]:
@ -623,7 +628,8 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_
from litellm.responses.main import _aresponses_websocket
with patch.object(
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
import_module("litellm.responses.main").base_llm_http_handler,
"async_responses_websocket",
new_callable=AsyncMock,
) as mock_ws:
await _aresponses_websocket(
@ -645,7 +651,8 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs(
from litellm.responses.main import _aresponses_websocket
with patch.object(
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
import_module("litellm.responses.main").base_llm_http_handler,
"async_responses_websocket",
new_callable=AsyncMock,
) as mock_ws:
await _aresponses_websocket(
@ -671,7 +678,9 @@ _ORIGINAL_WS_INPUT = [
@pytest.mark.asyncio
@pytest.mark.parametrize("nested", [False, True])
async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame(nested: bool): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket
async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame(
nested: bool,
): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket
from unittest.mock import MagicMock
from litellm.responses.main import _aresponses_websocket
@ -682,7 +691,8 @@ async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame
)
with patch.object(
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
import_module("litellm.responses.main").base_llm_http_handler,
"async_responses_websocket",
new_callable=AsyncMock,
) as mock_ws:
await _aresponses_websocket(
@ -711,7 +721,8 @@ async def test_aresponses_websocket_forwards_the_first_frame_verbatim_when_routi
first_message = '{"type": "response.create", "model": "gpt-5.6", "input": [{"role": "user", "content": "hi"}]}'
with patch.object(
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
import_module("litellm.responses.main").base_llm_http_handler,
"async_responses_websocket",
new_callable=AsyncMock,
) as mock_ws:
await _aresponses_websocket(
@ -968,9 +979,7 @@ async def test_injection_points_reach_a_foundry_deployment_of_an_openai_model(mo
Azure OpenAI one the Foundry deployment got Anthropic marks the Responses transform then stripped."""
monkeypatch.setenv("AZURE_AI_API_BASE", "https://other-deployment.openai.azure.com")
injected_client = AsyncHTTPHandler()
mock_post = AsyncMock(
return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200)
)
mock_post = AsyncMock(return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200))
injected_client.post = mock_post
await litellm.aresponses(
@ -1234,6 +1243,77 @@ def test_faked_stream_accepts_a_text_timeout(monkeypatch: pytest.MonkeyPatch) ->
]
async def _invoke_responses(sync_mode: bool, model: str, pricing: Mapping[str, float]) -> ResponsesAPIResponse:
response: Final = (
litellm.responses(model=model, input="hi", **pricing)
if sync_mode
else await litellm.aresponses(model=model, input="hi", **pricing)
)
assert isinstance(response, ResponsesAPIResponse)
return response
def _response_cost(response: ResponsesAPIResponse) -> float:
hidden_params: Final[dict[str, float]] = getattr(response, "_hidden_params")
return hidden_params["response_cost"]
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_mode", (True, False))
@pytest.mark.parametrize("model", ("openai/gpt-5.4-nano", "openai/custom-priced-model-not-in-cost-map"))
async def test_responses_bills_per_call_custom_pricing_from_usage(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, sync_mode: bool, model: str
):
monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key")
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost")
monkeypatch.setattr(litellm, "model_cost", dict(model_cost))
litellm.in_memory_llm_clients_cache.flush_cache()
respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_custom_price", model.split("/")[1]))
)
pricing: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5}
response: Final = await _invoke_responses(sync_mode, model, pricing)
assert response.usage is not None
expected: Final = (
response.usage.input_tokens * pricing["input_cost_per_token"]
+ response.usage.output_tokens * pricing["output_cost_per_token"]
)
assert math.isclose(_response_cost(response), expected)
@pytest.mark.asyncio
async def test_concurrent_responses_calls_keep_their_own_per_call_prices(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
):
monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key")
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
model: Final = "openai/gpt-5.4-nano"
model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost")
cost_map_row_before: Final = copy.deepcopy(model_cost.get(model))
respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_concurrent", "gpt-5.4-nano"))
)
rates: Final = ((1e-5, 2e-5), (3e-5, 4e-5))
responses: Final = await asyncio.gather(
*(
_invoke_responses(False, model, {"input_cost_per_token": in_rate, "output_cost_per_token": out_rate})
for in_rate, out_rate in rates
)
)
for response, (in_rate, out_rate) in zip(responses, rates, strict=True):
assert response.usage is not None
assert math.isclose(
_response_cost(response), response.usage.input_tokens * in_rate + response.usage.output_tokens * out_rate
)
assert model_cost.get(model) == cost_map_row_before
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize(
@ -1339,3 +1419,49 @@ async def test_responses_opencode_session_uses_effective_destination(
assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123")
assert sent.headers["authorization"] == "Bearer test-key"
assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello"}
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
@pytest.mark.asyncio
async def test_router_responses_deployment_pricing_keeps_inherited_cache_rates(
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost")
monkeypatch.setattr(litellm, "model_cost", dict(model_cost))
litellm.in_memory_llm_clients_cache.flush_cache()
rates: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5}
router: Final = litellm.Router(
model_list=[
{
"model_name": "nano",
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake-api-key", **rates},
"model_info": {"id": "nano-deployment"},
}
]
)
payload: Final[dict[str, object]] = _minimal_responses_api_payload("resp_router_cache", "gpt-5.4-nano")
usage: Final = _JSON_OBJECT.validate_python(payload["usage"])
cached_tokens: Final = 8
respx_mock.post("https://api.openai.com/v1/responses").mock(
return_value=httpx.Response(
200,
json={**payload, "usage": {**usage, "input_tokens_details": {"cached_tokens": cached_tokens}}},
)
)
response: Final = await router.aresponses(model="nano", input="hi")
assert isinstance(response, ResponsesAPIResponse)
assert response.usage is not None
uncached_input: Final = response.usage.input_tokens - cached_tokens
full_price: Final = (
response.usage.input_tokens * rates["input_cost_per_token"]
+ response.usage.output_tokens * rates["output_cost_per_token"]
)
floor: Final = (
uncached_input * rates["input_cost_per_token"] + response.usage.output_tokens * rates["output_cost_per_token"]
)
assert floor < _response_cost(response) < full_price

View file

@ -1,5 +1,6 @@
from importlib import import_module
import base64
from typing import Final
from unittest.mock import MagicMock, patch
import pytest
@ -7,10 +8,11 @@ import pytest
import litellm
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
from litellm.types.utils import Usage
from litellm.types.utils import ModelInfo, Usage
from litellm.utils import _invalidate_model_cost_lowercase_map
@ -413,6 +415,43 @@ class TestResponseAPILoggingUtils:
assert result.prompt_tokens_details.cache_creation_tokens == 10059
assert result.prompt_tokens_details.cached_tokens == 0
@pytest.mark.parametrize("as_model", [False, True])
def test_transform_response_api_usage_prices_12h_cache_writes(self, as_model: bool) -> None:
usage_data: Final = {
"input_tokens": 5_000,
"output_tokens": 0,
"total_tokens": 5_000,
"input_tokens_details": {
"cache_write_tokens": 5_000,
"cache_write_tokens_details": {"ttl_30m_tokens": 3_000, "ttl_12h_tokens": 2_000},
},
}
usage_input: Final = ResponseAPIUsage(**usage_data) if as_model else usage_data
usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_input)
model_info: Final[ModelInfo] = {
"input_cost_per_token": 1e-6,
"output_cost_per_token": 2e-6,
"cache_creation_input_token_cost": 3e-6,
"cache_creation_input_token_cost_above_12hr": 5e-6,
}
assert usage.prompt_tokens_details is not None
assert usage.prompt_tokens_details.cache_write_tokens == 5_000
assert usage.prompt_tokens_details.cache_creation_token_details is not None
assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_12h_input_tokens == 2_000
assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_30m_input_tokens == 3_000
cost: Final = generic_cost_per_token(
model="test-12h-cache-write",
usage=usage,
custom_llm_provider="openai",
model_info=model_info,
)[0]
assert cost == pytest.approx(
3_000 * model_info["cache_creation_input_token_cost"]
+ 2_000 * model_info["cache_creation_input_token_cost_above_12hr"]
)
def test_transform_response_api_usage_mixed_details(self):
"""Test transformation handles mixed token details (cached + image + audio)."""
# Setup - hypothetical usage with mixed token types

View file

@ -16,6 +16,7 @@ import litellm
from litellm import TranscriptionResponse, model_cost
from litellm.constants import REALTIME_METERED_USAGE_KEY
from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.fallback_generalizations import (
get_fallback_generalization_rules,
@ -3337,14 +3338,7 @@ def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(_loca
def test_gemini_cache_tokens_details_no_negative_values():
"""
Test for Issue #18750: Negative text_tokens with Gemini caching
When using Gemini with explicit caching, the response includes cacheTokensDetails
which breaks down cached tokens by modality. This test ensures that:
1. text_tokens is never negative
2. We correctly subtract cached tokens per modality (not total)
"""
"""Ensure issue #18750 cache details remain non-negative after cost parsing."""
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
@ -3372,27 +3366,17 @@ def test_gemini_cache_tokens_details_no_negative_values():
usage = VertexGeminiConfig.calculate_usage(completion_response)
# Text tokens should be non-cached text only: 9402 - 9393 = 9
assert usage.prompt_tokens_details.text_tokens == 9, (
f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}"
)
prompt_details: Final = parse_prompt_tokens_details(usage)
# Image tokens should be non-cached image only: 258 - 258 = 0
assert usage.prompt_tokens_details.image_tokens == 0, (
f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}"
assert usage.prompt_tokens_details.text_tokens == 9402
assert usage.prompt_tokens_details.image_tokens == 258
assert usage.prompt_tokens_details.cached_tokens == 9651
assert usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(
text_tokens=9393, image_tokens=258
)
# Total cached should match
assert usage.prompt_tokens_details.cached_tokens == 9651, (
f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}"
)
# MOST IMPORTANT: text_tokens should NEVER be negative
assert usage.prompt_tokens_details.text_tokens >= 0, (
f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750"
)
print("✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative")
assert prompt_details["text_tokens"] == 9
assert prompt_details["image_tokens"] == 0
assert prompt_details["text_tokens"] >= 0
def test_gemini_without_cache_tokens_details():
@ -5990,6 +5974,45 @@ def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_t
assert prompt_cost == pytest.approx(298_926 * 3e-6 + 64 * 5e-6 + 10 * 3e-6 + 1_000 * 3e-7)
def test_batch_cost_calculator_prices_cached_audio_at_its_batch_rate() -> None:
model_info: Final[ModelInfo] = {
"input_cost_per_token": 2e-6,
"input_cost_per_token_batches": 1e-6,
"input_cost_per_audio_token": 6e-6,
"input_cost_per_audio_token_batches": 3e-6,
"cache_read_input_token_cost": 8e-7,
"cache_read_input_token_cost_batches": 4e-7,
"cache_read_input_audio_token_cost": 5e-7,
"cache_read_input_audio_token_cost_batches": 2e-7,
}
usage: Final = Usage(
prompt_tokens=40,
completion_tokens=0,
total_tokens=40,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=20,
audio_tokens=20,
cached_tokens=10,
cached_tokens_details=CachedTokensDetails(text_tokens=5, audio_tokens=5),
),
)
prompt_cost, _ = batch_cost_calculator(
usage=usage,
model="gemini-audio-batch-test-model",
custom_llm_provider="gemini",
model_info=model_info,
)
expected: Final = (
15 * model_info["input_cost_per_token_batches"]
+ 15 * model_info["input_cost_per_audio_token_batches"]
+ 5 * model_info["cache_read_input_token_cost_batches"]
+ 5 * model_info["cache_read_input_audio_token_cost_batches"]
)
assert prompt_cost == pytest.approx(expected)
QWEN3_NEXT_REGIONS: Final = ("ap-northeast-1", "ap-south-1", "ap-southeast-2", "eu-west-1", "eu-west-2", "sa-east-1")

View file

@ -791,3 +791,26 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa
sibling = rows["volcengine/doubao-seed-2-1-pro-260628"]
assert isinstance(sibling, dict)
assert all(entry[key] == sibling[key] for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost"))
def _bedrock_gpt_oss_rows(prices: Mapping[str, JsonValue]) -> tuple[str, ...]:
return tuple(
key
for key, row in prices.items()
if "gpt-oss" in key and isinstance(row, dict) and str(row.get("litellm_provider", "")).startswith("bedrock")
)
@pytest.mark.parametrize("path", [PRICES_PATH, BACKUP_PRICES_PATH], ids=["main", "backup"])
def test_bedrock_gpt_oss_rows_declare_vision_support_like_their_safeguard_siblings(path: Path) -> None:
prices: Final = json.loads(path.read_text())
rows: Final = _bedrock_gpt_oss_rows(prices)
assert rows, "no Bedrock gpt-oss rows found"
reference: Final = prices["openai.gpt-oss-safeguard-120b"]["supports_vision"]
undeclared: Final = tuple(key for key in rows if prices[key].get("supports_vision") is not reference)
assert undeclared == (), f"modality routing only skips rows that declare supports_vision: {undeclared}"
def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> None:
assert litellm.utils.is_vision_explicitly_disabled("bedrock/openai.gpt-oss-120b-1:0") is True
assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True

View file

@ -29871,3 +29871,105 @@ async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_s
wrapped.adopt_fallback_headers(fallback)
assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens
_IMAGE_PART = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}
_IMAGE_CHAT = [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, _IMAGE_PART]}]
_IMAGE_RESPONSES_INPUT = [
{
"role": "user",
"content": [
{"type": "input_text", "text": "What is in this image?"},
{"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo="},
],
}
]
_VISION_LEG: Final[dict[str, object]] = {"litellm_params": {"model": "openai/gpt-5.6-terra"}, "model_info": {"id": "vision"}}
_DECLARED_TEXT_LEG: Final[dict[str, object]] = {
"litellm_params": {"model": "bedrock/openai.gpt-oss-120b-1:0"},
"model_info": {"id": "declared-text", "supports_vision": False},
}
def _pre_call_router(*deployments: dict[str, object], routing_strategy: str = "latency-based-routing") -> litellm.Router:
return litellm.Router(
model_list=[{"model_name": "mixed", **deployment} for deployment in deployments],
routing_strategy=routing_strategy,
enable_pre_call_checks=True,
)
def _pre_call_ids(router: litellm.Router, deployments: tuple[dict[str, object], ...], **request: object) -> list[str]:
return [d["model_info"]["id"] for d in router._pre_call_checks(model="mixed", healthy_deployments=list(deployments), **request)]
def test_pre_call_checks_drop_declared_text_only_legs_for_image_requests():
router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG)
assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=_IMAGE_CHAT) == ["vision"]
def test_pre_call_checks_keep_declared_text_only_legs_for_text_requests():
router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG)
assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=[{"role": "user", "content": "Say hi"}]) == [
"vision",
"declared-text",
]
def test_pre_call_checks_drop_declared_text_only_legs_for_responses_input_image():
router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG)
assert _pre_call_ids(
router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=None, request_kwargs={"input": _IMAGE_RESPONSES_INPUT}
) == ["vision"]
def test_pre_call_checks_send_image_to_the_group_when_every_leg_is_declared_text_only():
other_text_leg = {**_DECLARED_TEXT_LEG, "model_info": {"id": "declared-text-2", "supports_vision": False}}
router = _pre_call_router(_DECLARED_TEXT_LEG, other_text_leg)
assert _pre_call_ids(router, (_DECLARED_TEXT_LEG, other_text_leg), messages=_IMAGE_CHAT) == [
"declared-text",
"declared-text-2",
]
def test_pre_call_checks_keep_undeclared_custom_legs_for_image_requests():
custom_leg = {
"litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct", "api_base": "http://localhost:8000"},
"model_info": {"id": "custom"},
}
router = _pre_call_router(_VISION_LEG, custom_leg)
assert _pre_call_ids(router, (_VISION_LEG, custom_leg), messages=_IMAGE_CHAT) == ["vision", "custom"]
def test_pre_call_checks_resolve_vision_through_the_cost_map_and_base_model():
catalog_text_leg = {"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {"id": "catalog-text"}}
azure_text_leg = {
"litellm_params": {"model": "azure/my-audio-deployment", "api_base": "https://x.openai.azure.com"},
"model_info": {"id": "azure-text", "base_model": "azure/gpt-audio"},
}
router = _pre_call_router(_VISION_LEG, catalog_text_leg, azure_text_leg)
assert _pre_call_ids(router, (_VISION_LEG, catalog_text_leg, azure_text_leg), messages=_IMAGE_CHAT) == ["vision"]
@pytest.mark.parametrize(
("deployment", "expected"),
[
(_DECLARED_TEXT_LEG, True),
({"litellm_params": {"model": "bedrock/openai.gpt-oss-120b-1:0"}, "model_info": {"supports_vision": True}}, False),
({"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {}}, True),
({"litellm_params": {"model": "azure/my-deploy"}, "model_info": {"base_model": "azure/gpt-audio"}}, True),
({"litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct"}, "model_info": {}}, False),
({"litellm_params": {"model": "no-such-model-anywhere"}, "model_info": {}}, False),
({"litellm_params": {}, "model_info": {}}, False),
],
ids=["override-false", "override-true", "cost-map-false", "base-model-false", "undeclared", "unresolvable", "no-params"],
)
def test_deployment_declares_no_vision_reads_the_override_then_the_cost_map(deployment: dict[str, object], expected: bool):
assert litellm.Router._deployment_declares_no_vision(deployment, "mixed") is expected
def test_request_carries_image_sees_chat_parts_and_responses_input_items():
assert litellm.Router._request_carries_image(_IMAGE_CHAT, {}) is True
assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True
assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False
assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False

View file

@ -939,6 +939,18 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"cache_creation_input_audio_token_cost": {"type": "number"},
"cache_creation_input_token_cost": {"type": "number"},
"cache_creation_input_token_cost_above_1hr": {"type": "number"},
"cache_creation_input_token_cost_above_12hr": {"type": "number"},
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": {
"type": "number"
},
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": {
"type": "number"
},
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": {
"type": "number"
},
"cache_creation_input_token_cost_above_12hr_flex": {"type": "number"},
"cache_creation_input_token_cost_above_12hr_priority": {"type": "number"},
"cache_creation_input_token_cost_above_32k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_100k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_100k_tokens_batches": {"type": "number"},
@ -982,6 +994,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"},
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"},
"cache_read_input_audio_token_cost": {"type": "number"},
"cache_read_input_audio_token_cost_batches": {"type": "number"},
"cache_read_input_audio_token_cost_flex": {"type": "number"},
"cache_read_input_audio_token_cost_priority": {"type": "number"},
"cache_read_input_image_token_cost": {"type": "number"},
"audio_transcription_config": {"type": "string"},
"deprecation_date": {"type": "string"},
@ -989,6 +1004,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"},
"google_maps_grounding_cost_per_query": {"type": "number"},
"input_cost_per_audio_token": {"type": "number"},
"input_cost_per_audio_token_flex": {"type": "number"},
"input_cost_per_image_token": {"type": "number"},
"input_cost_per_character": {"type": "number"},
"input_cost_per_character_above_128k_tokens": {"type": "number"},

View file

@ -61,6 +61,30 @@ def test_get_model_info_custom_llm_with_same_name_vllm(monkeypatch):
assert model_info["input_cost_per_token"] == 0.0
def test_get_model_info_passes_through_gemini_tiered_audio_rates() -> None:
model: Final = "gemini-tiered-audio-rates-test"
prices: Final = {
"input_cost_per_audio_token_flex": 1e-6,
"input_cost_per_audio_token_priority": 2e-6,
"cache_read_input_audio_token_cost_flex": 3e-7,
"cache_read_input_audio_token_cost_priority": 4e-7,
"cache_read_input_audio_token_cost_batches": 5e-7,
}
litellm.register_model(
{
model: {
"litellm_provider": "gemini",
"mode": "chat",
**prices,
}
}
)
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="gemini")
assert {key: model_info[key] for key in prices} == prices
def test_get_model_info_ollama_chat():
from litellm.llms.ollama.completion.transformation import OllamaConfig

View file

@ -28439,6 +28439,15 @@ export interface components {
*/
uncached_input_tokens: number;
};
/** CacheWriteTokensDetails */
CacheWriteTokensDetails: {
/** Ttl 12H Tokens */
ttl_12h_tokens: number;
/** Ttl 30M Tokens */
ttl_30m_tokens: number;
} & {
[key: string]: unknown;
};
/** CachedTokensDetails */
CachedTokensDetails: {
/** Audio Tokens */
@ -33589,6 +33598,9 @@ export interface components {
InputTokensDetails: {
/** Audio Tokens */
audio_tokens?: number | null;
/** Cache Write Tokens */
cache_write_tokens?: number | null;
cache_write_tokens_details?: components["schemas"]["CacheWriteTokensDetails"] | null;
/**
* Cached Tokens
* @default 0
@ -35207,6 +35219,18 @@ export interface components {
cache_creation_input_token_cost_above_100k_tokens?: number | null;
/** Cache Creation Input Token Cost Above 100K Tokens Batches */
cache_creation_input_token_cost_above_100k_tokens_batches?: number | null;
/** Cache Creation Input Token Cost Above 12Hr */
cache_creation_input_token_cost_above_12hr?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */
cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Flex */
cache_creation_input_token_cost_above_12hr_flex?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Priority */
cache_creation_input_token_cost_above_12hr_priority?: number | null;
/** Cache Creation Input Token Cost Above 1Hr */
cache_creation_input_token_cost_above_1hr?: number | null;
/** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */
@ -35235,6 +35259,12 @@ export interface components {
cache_creation_input_token_cost_ultrafast?: number | null;
/** Cache Read Input Audio Token Cost */
cache_read_input_audio_token_cost?: number | null;
/** Cache Read Input Audio Token Cost Batches */
cache_read_input_audio_token_cost_batches?: number | null;
/** Cache Read Input Audio Token Cost Flex */
cache_read_input_audio_token_cost_flex?: number | null;
/** Cache Read Input Audio Token Cost Priority */
cache_read_input_audio_token_cost_priority?: number | null;
/** Cache Read Input Image Token Cost */
cache_read_input_image_token_cost?: number | null;
/** Cache Read Input Token Cost */
@ -35314,6 +35344,10 @@ export interface components {
input_cost_per_audio_token?: number | null;
/** Input Cost Per Audio Token Batches */
input_cost_per_audio_token_batches?: number | null;
/** Input Cost Per Audio Token Flex */
input_cost_per_audio_token_flex?: number | null;
/** Input Cost Per Audio Token Priority */
input_cost_per_audio_token_priority?: number | null;
/** Input Cost Per Character */
input_cost_per_character?: number | null;
/** Input Cost Per Character Above 128K Tokens */
@ -41506,9 +41540,9 @@ export interface components {
mode?: "implicit" | "explicit";
/**
* Ttl
* @constant
* @enum {string}
*/
ttl?: "30m";
ttl?: "30m" | "12h";
};
/** PromptCachingRequest */
PromptCachingRequest: {
@ -50199,6 +50233,18 @@ export interface components {
blocked?: boolean | null;
/** Cache Creation Input Token Cost */
cache_creation_input_token_cost?: number | null;
/** Cache Creation Input Token Cost Above 12Hr */
cache_creation_input_token_cost_above_12hr?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */
cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Flex */
cache_creation_input_token_cost_above_12hr_flex?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Priority */
cache_creation_input_token_cost_above_12hr_priority?: number | null;
/** Cache Creation Input Token Cost Above 1Hr */
cache_creation_input_token_cost_above_1hr?: number | null;
/** Cache Read Input Token Cost */
@ -50492,6 +50538,18 @@ export interface components {
cache_creation_input_token_cost_above_100k_tokens?: number | null;
/** Cache Creation Input Token Cost Above 100K Tokens Batches */
cache_creation_input_token_cost_above_100k_tokens_batches?: number | null;
/** Cache Creation Input Token Cost Above 12Hr */
cache_creation_input_token_cost_above_12hr?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */
cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Flex */
cache_creation_input_token_cost_above_12hr_flex?: number | null;
/** Cache Creation Input Token Cost Above 12Hr Priority */
cache_creation_input_token_cost_above_12hr_priority?: number | null;
/** Cache Creation Input Token Cost Above 1Hr */
cache_creation_input_token_cost_above_1hr?: number | null;
/** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */
@ -50520,6 +50578,12 @@ export interface components {
cache_creation_input_token_cost_ultrafast?: number | null;
/** Cache Read Input Audio Token Cost */
cache_read_input_audio_token_cost?: number | null;
/** Cache Read Input Audio Token Cost Batches */
cache_read_input_audio_token_cost_batches?: number | null;
/** Cache Read Input Audio Token Cost Flex */
cache_read_input_audio_token_cost_flex?: number | null;
/** Cache Read Input Audio Token Cost Priority */
cache_read_input_audio_token_cost_priority?: number | null;
/** Cache Read Input Image Token Cost */
cache_read_input_image_token_cost?: number | null;
/** Cache Read Input Token Cost */
@ -50599,6 +50663,10 @@ export interface components {
input_cost_per_audio_token?: number | null;
/** Input Cost Per Audio Token Batches */
input_cost_per_audio_token_batches?: number | null;
/** Input Cost Per Audio Token Flex */
input_cost_per_audio_token_flex?: number | null;
/** Input Cost Per Audio Token Priority */
input_cost_per_audio_token_priority?: number | null;
/** Input Cost Per Character */
input_cost_per_character?: number | null;
/** Input Cost Per Character Above 128K Tokens */