mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
Merge remote-tracking branch 'origin/main' into HEAD
# Conflicts: # litellm/responses/utils.py
This commit is contained in:
commit
9dfcb67470
33 changed files with 1646 additions and 302 deletions
|
|
@ -2268,6 +2268,8 @@ def response_cost_calculator(
|
|||
### VERTEX LOCATION ###
|
||||
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
|
||||
region_name: str | None = None,
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Returns
|
||||
|
|
@ -2296,6 +2298,8 @@ def response_cost_calculator(
|
|||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
|
|
@ -2879,6 +2883,7 @@ def _batch_rate(
|
|||
model_info: ModelInfo,
|
||||
key: Literal[
|
||||
"input_cost_per_audio_token_batches",
|
||||
"cache_read_input_audio_token_cost_batches",
|
||||
"input_cost_per_image_token_batches",
|
||||
"input_cost_per_video_token_batches",
|
||||
],
|
||||
|
|
@ -3012,6 +3017,7 @@ def _batch_prompt_cost(
|
|||
) -> float:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
|
||||
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
|
||||
audio_tokens, image_tokens, video_tokens = (
|
||||
details["audio_tokens"],
|
||||
|
|
@ -3026,7 +3032,9 @@ def _batch_prompt_cost(
|
|||
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
|
||||
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
|
||||
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
|
||||
+ cached_tokens * (cache_read_rate or 0.0)
|
||||
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
|
||||
+ cached_audio_tokens
|
||||
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
|
||||
+ written_tokens * (cache_creation_rate or 0.0)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -161,6 +161,7 @@ from litellm.types.utils import (
|
|||
CachingDetails,
|
||||
CallTypes,
|
||||
CostBreakdown,
|
||||
CostPerToken,
|
||||
CustomPricingLiteLLMParams,
|
||||
DynamicPromptManagementParamLiteral,
|
||||
EmbeddingResponse,
|
||||
|
|
@ -622,6 +623,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
prometheusLogger, \
|
||||
slack_app
|
||||
custom_pricing: bool = False
|
||||
custom_cost_per_token: CostPerToken | None = None
|
||||
custom_cost_per_second: float | None = None
|
||||
replaced_global_callback_keys: frozenset[str] = frozenset()
|
||||
stream_options = None
|
||||
litellm_request_debug: bool = False
|
||||
|
|
@ -2097,6 +2100,8 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"call_type": self.call_type,
|
||||
"optional_params": self.optional_params,
|
||||
"custom_pricing": custom_pricing,
|
||||
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
|
||||
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
|
||||
"prompt": prompt,
|
||||
"standard_built_in_tools_params": self.standard_built_in_tools_params,
|
||||
"router_model_id": router_model_id,
|
||||
|
|
|
|||
|
|
@ -5,30 +5,42 @@ Shared by provider cost calculators (e.g. Dashscope) and the proxy budget
|
|||
reservation logic so neither has to depend on the other.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, SupportsFloat
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
_TIER_RANGE: Final = TypeAdapter(tuple[float, float])
|
||||
|
||||
|
||||
def _coerce_cost_per_token(value: float | str | None) -> float:
|
||||
def _coerce_cost_per_token(value: object) -> float:
|
||||
"""
|
||||
Coerce a per-token cost into a float.
|
||||
|
||||
Model cost values loaded from YAML config may arrive as strings (e.g.
|
||||
scientific notation like "4e-07"), which would break arithmetic.
|
||||
"""
|
||||
if value is None:
|
||||
return 0.0
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
return float(value)
|
||||
if isinstance(value, SupportsFloat):
|
||||
return float(value)
|
||||
return 0.0
|
||||
|
||||
|
||||
def _tier_range(tier: Mapping[str, object]) -> tuple[float, float] | None:
|
||||
try:
|
||||
return _TIER_RANGE.validate_python(tier.get("range"))
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def select_tier_for_input(
|
||||
tiered_pricing: list[dict],
|
||||
tiered_pricing: Sequence[Mapping[str, object]],
|
||||
input_tokens: int,
|
||||
) -> dict | None:
|
||||
) -> Mapping[str, object] | None:
|
||||
"""
|
||||
Select the pricing tier for a request based on its total input token count.
|
||||
|
||||
|
|
@ -44,19 +56,21 @@ def select_tier_for_input(
|
|||
if not tiered_pricing or input_tokens <= 0:
|
||||
return None
|
||||
|
||||
sorted_tiers: Final = sorted(tiered_pricing, key=lambda t: t.get("range", [0, 0])[0])
|
||||
valid_tiers: Final = [tier for tier in sorted_tiers if len(tier.get("range", [])) == 2]
|
||||
if not valid_tiers:
|
||||
ranged_tiers: Final = sorted(
|
||||
((bounds, tier) for tier in tiered_pricing if (bounds := _tier_range(tier)) is not None),
|
||||
key=lambda ranged: ranged[0][0],
|
||||
)
|
||||
if not ranged_tiers:
|
||||
return None
|
||||
|
||||
matching: Final = [tier for tier in valid_tiers if tier["range"][0] < input_tokens <= tier["range"][1]]
|
||||
matching: Final = tuple(tier for (start, end), tier in ranged_tiers if start < input_tokens <= end)
|
||||
if matching:
|
||||
return matching[0]
|
||||
return valid_tiers[-1]
|
||||
return ranged_tiers[-1][1]
|
||||
|
||||
|
||||
def tier_rate(
|
||||
tier: dict,
|
||||
tier: Mapping[str, object],
|
||||
cost_key: str,
|
||||
fallback_cost_key: str | None = None,
|
||||
) -> float:
|
||||
|
|
@ -68,4 +82,6 @@ def tier_rate(
|
|||
primary: Final = tier.get(cost_key)
|
||||
if primary is not None:
|
||||
return _coerce_cost_per_token(primary)
|
||||
if fallback_cost_key is None:
|
||||
return 0.0
|
||||
return _coerce_cost_per_token(tier.get(fallback_cost_key, 0))
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime, timezone, tzinfo
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypedDict, cast
|
||||
|
|
@ -52,6 +52,22 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency)
|
|||
|
||||
_DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields)
|
||||
|
||||
_BASE_CACHE_WRITE_RATE_KEY: Final = "cache_creation_input_token_cost"
|
||||
|
||||
_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"5m": _BASE_CACHE_WRITE_RATE_KEY,
|
||||
"30m": _BASE_CACHE_WRITE_RATE_KEY,
|
||||
"1hr": "cache_creation_input_token_cost_above_1hr",
|
||||
"12hr": "cache_creation_input_token_cost_above_12hr",
|
||||
}
|
||||
)
|
||||
|
||||
_TTL_PRICED_CACHE_WRITE_RATE_KEYS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{ttl: key for ttl, key in _CACHE_WRITE_RATE_KEYS.items() if key != _BASE_CACHE_WRITE_RATE_KEY}
|
||||
)
|
||||
|
||||
|
||||
_IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
|
|
@ -356,7 +372,21 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide
|
|||
)
|
||||
|
||||
|
||||
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict[str, object] | None:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TokenRates:
|
||||
input_rate: float
|
||||
output_rate: float
|
||||
cache_read_rate: float
|
||||
cache_creation_rate: float
|
||||
reasoning_rate: float | None
|
||||
cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({}))
|
||||
|
||||
@property
|
||||
def billed_reasoning_rate(self) -> float:
|
||||
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
|
||||
|
||||
|
||||
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, object] | None:
|
||||
tiered_pricing: Final = model_info.get("tiered_pricing")
|
||||
if not isinstance(tiered_pricing, list) or not tiered_pricing:
|
||||
return None
|
||||
|
|
@ -376,7 +406,7 @@ def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | N
|
|||
return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token")
|
||||
|
||||
|
||||
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float, float, float, float, float] | None:
|
||||
def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None:
|
||||
"""
|
||||
Resolve the base rates from a model's ``tiered_pricing`` table, if it has one.
|
||||
|
||||
|
|
@ -391,22 +421,20 @@ def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[float,
|
|||
if tier is None:
|
||||
return None
|
||||
|
||||
cache_creation_cost: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
|
||||
completion_cost: Final = (
|
||||
tier_rate(tier, "output_cost_per_token")
|
||||
if "output_cost_per_token" in tier
|
||||
else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0
|
||||
)
|
||||
return (
|
||||
tier_rate(tier, "input_cost_per_token"),
|
||||
completion_cost,
|
||||
cache_creation_cost,
|
||||
tier_rate(tier, "cache_creation_input_token_cost_above_1hr")
|
||||
if "cache_creation_input_token_cost_above_1hr" in tier
|
||||
and tier["cache_creation_input_token_cost_above_1hr"] is not None
|
||||
else cache_creation_cost,
|
||||
tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
|
||||
cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
|
||||
rates: Final = TokenRates(
|
||||
input_rate=tier_rate(tier, "input_cost_per_token"),
|
||||
output_rate=completion_cost,
|
||||
cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
|
||||
cache_creation_rate=cache_creation_rate,
|
||||
reasoning_rate=None,
|
||||
)
|
||||
return rates, tier
|
||||
|
||||
|
||||
def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool:
|
||||
|
|
@ -532,19 +560,6 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None =
|
|||
return False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TokenRates:
|
||||
input_rate: float
|
||||
output_rate: float
|
||||
cache_read_rate: float
|
||||
cache_creation_rate: float
|
||||
reasoning_rate: float | None
|
||||
|
||||
@property
|
||||
def billed_reasoning_rate(self) -> float:
|
||||
return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
|
||||
|
||||
|
||||
def resolve_explicit_cache_read_cost_rate(
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
|
|
@ -649,7 +664,8 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
|
|||
if off_peak is None:
|
||||
return rates
|
||||
off_peak_reasoning_rate: Final = _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token"))
|
||||
return TokenRates(
|
||||
return replace(
|
||||
rates,
|
||||
input_rate=_off_peak_rate(off_peak, "input_cost_per_token", rates.input_rate),
|
||||
output_rate=_off_peak_rate(off_peak, "output_cost_per_token", rates.output_rate),
|
||||
cache_read_rate=_off_peak_rate(off_peak, "cache_read_input_token_cost", rates.cache_read_rate),
|
||||
|
|
@ -661,32 +677,28 @@ def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None,
|
|||
def _apply_off_peak_to_base_costs(
|
||||
model_info: ModelInfo,
|
||||
current_time: datetime | None,
|
||||
base_costs: tuple[float, float, float, float | None, float],
|
||||
) -> tuple[float, float, float, float, float]:
|
||||
"""Apply off-peak rates to an already-resolved set of base costs, whichever pricing path
|
||||
produced them. off_peak_pricing has no field for the one-hour cache-creation rate, so a
|
||||
present one passes through untouched and an absent one resolves to the applied
|
||||
cache-creation rate. Reasoning is left to _resolve_billed_reasoning_rate.
|
||||
base_rates: TokenRates,
|
||||
tier: Mapping[str, object],
|
||||
) -> TokenRates:
|
||||
"""Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no
|
||||
field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier
|
||||
that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to
|
||||
_resolve_billed_reasoning_rate.
|
||||
"""
|
||||
prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
|
||||
rates: Final = apply_off_peak_pricing(
|
||||
model_info,
|
||||
current_time,
|
||||
TokenRates(
|
||||
input_rate=prompt,
|
||||
output_rate=completion,
|
||||
cache_read_rate=cache_read,
|
||||
cache_creation_rate=cache_creation,
|
||||
reasoning_rate=None,
|
||||
rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates)
|
||||
return replace(
|
||||
rates,
|
||||
cache_creation_rates_by_ttl=MappingProxyType(
|
||||
{
|
||||
ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key)
|
||||
for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()
|
||||
}
|
||||
),
|
||||
)
|
||||
return (
|
||||
rates.input_rate,
|
||||
rates.output_rate,
|
||||
rates.cache_creation_rate,
|
||||
rates.cache_creation_rate if cache_creation_above_1hr is None else cache_creation_above_1hr,
|
||||
rates.cache_read_rate,
|
||||
)
|
||||
|
||||
|
||||
def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str:
|
||||
return preferred_key if model_info.get(preferred_key) is not None else fallback_key
|
||||
|
||||
|
||||
def _get_token_base_cost(
|
||||
|
|
@ -696,7 +708,7 @@ def _get_token_base_cost(
|
|||
current_time: datetime | None = None,
|
||||
*,
|
||||
threshold_is_inclusive: bool = False,
|
||||
) -> tuple[float, float, float, float, float]:
|
||||
) -> TokenRates:
|
||||
"""
|
||||
Return prompt cost, completion cost, and cache costs for a given model and usage.
|
||||
|
||||
|
|
@ -709,15 +721,12 @@ def _get_token_base_cost(
|
|||
An absent cache-creation or cache-read rate always resolves to the resolved input
|
||||
rate, the way the tiered table and custom deployment pricing already do, since a
|
||||
provider that publishes no cache price bills cached tokens as ordinary input. An
|
||||
absent 1h write rate resolves to the cache-creation rate, off-peak included. An
|
||||
absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An
|
||||
explicit 0.0 stays a real price for all of them.
|
||||
|
||||
Returns:
|
||||
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
|
||||
"""
|
||||
tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage)
|
||||
if tiered_base_costs is not None:
|
||||
return _apply_off_peak_to_base_costs(model_info, current_time, tiered_base_costs)
|
||||
return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs)
|
||||
|
||||
# Get service tier aware cost keys
|
||||
input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier)
|
||||
|
|
@ -734,13 +743,13 @@ def _get_token_base_cost(
|
|||
|
||||
# For image generation models that don't have output_cost_per_token,
|
||||
# use output_cost_per_image_token as the base cost (all output tokens are image tokens)
|
||||
if completion_base_cost == 0.0 or completion_base_cost is None:
|
||||
if completion_base_cost == 0.0:
|
||||
output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None)
|
||||
if output_image_cost is not None:
|
||||
completion_base_cost = output_image_cost
|
||||
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None)
|
||||
cache_creation_cost_above_1hr = get_cost_per_unit(
|
||||
model_info, "cache_creation_input_token_cost_above_1hr", default_value=None
|
||||
cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType(
|
||||
{ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()}
|
||||
)
|
||||
cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None)
|
||||
|
||||
|
|
@ -811,14 +820,6 @@ def _get_token_base_cost(
|
|||
if service_tier
|
||||
else f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
|
||||
)
|
||||
cache_creation_1hr_tiered_key = (
|
||||
get_service_tier_cost_key(
|
||||
f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
|
||||
)
|
||||
cache_read_tiered_key = (
|
||||
get_service_tier_cost_key(
|
||||
f"cache_read_input_token_cost_above_{threshold_str}_tokens",
|
||||
|
|
@ -830,10 +831,19 @@ def _get_token_base_cost(
|
|||
|
||||
cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost)
|
||||
|
||||
cache_creation_cost_above_1hr = get_cost_per_unit(
|
||||
model_info, cache_creation_1hr_tiered_key, cache_creation_cost_above_1hr
|
||||
cache_creation_keys_by_ttl = MappingProxyType(
|
||||
{
|
||||
ttl: _first_present_key(
|
||||
model_info,
|
||||
get_service_tier_cost_key(
|
||||
f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
),
|
||||
key,
|
||||
)
|
||||
for ttl, key in cache_creation_keys_by_ttl.items()
|
||||
}
|
||||
)
|
||||
|
||||
cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost)
|
||||
except (IndexError, ValueError):
|
||||
continue
|
||||
|
|
@ -849,16 +859,26 @@ def _get_token_base_cost(
|
|||
resolved_cache_creation_cost: Final = (
|
||||
input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost
|
||||
)
|
||||
|
||||
return _apply_off_peak_to_base_costs(
|
||||
base_rates: Final = apply_off_peak_pricing(
|
||||
model_info,
|
||||
current_time,
|
||||
(
|
||||
prompt_base_cost,
|
||||
completion_base_cost,
|
||||
resolved_cache_creation_cost,
|
||||
cache_creation_cost_above_1hr,
|
||||
resolved_cache_read_cost,
|
||||
TokenRates(
|
||||
input_rate=prompt_base_cost,
|
||||
output_rate=completion_base_cost,
|
||||
cache_read_rate=resolved_cache_read_cost,
|
||||
cache_creation_rate=resolved_cache_creation_cost,
|
||||
reasoning_rate=None,
|
||||
),
|
||||
)
|
||||
return replace(
|
||||
base_rates,
|
||||
cache_creation_rates_by_ttl=MappingProxyType(
|
||||
{
|
||||
ttl: base_rates.cache_creation_rate
|
||||
if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None
|
||||
else rate
|
||||
for ttl, key in cache_creation_keys_by_ttl.items()
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -975,25 +995,28 @@ def resolve_image_model_info(model: str, custom_llm_provider: str, model_info: M
|
|||
def calculate_cache_writing_cost(
|
||||
cache_creation_tokens: int,
|
||||
cache_creation_token_details: CacheCreationTokenDetails | None,
|
||||
cache_creation_cost_above_1hr: float,
|
||||
cache_creation_cost: float,
|
||||
cache_creation_cost_by_ttl: Mapping[str, float],
|
||||
) -> float:
|
||||
"""
|
||||
Adjust cost of cache creation tokens based on the cache creation token details.
|
||||
Bill cache writes per lifetime: Anthropic reports 5m and 1h counts, OpenAI reports 30m and 12h counts.
|
||||
Writes no lifetime count covers, such as a lifetime a provider adds later, bill at the base write rate.
|
||||
"""
|
||||
total_cost: float = 0.0
|
||||
if cache_creation_token_details is not None:
|
||||
# get the number of 5m and 1h cache creation tokens
|
||||
cache_creation_tokens_5m: Final = cache_creation_token_details.ephemeral_5m_input_tokens
|
||||
cache_creation_tokens_1h: Final = cache_creation_token_details.ephemeral_1h_input_tokens
|
||||
# add the number of 5m and 1h cache creation tokens to the cache creation tokens
|
||||
total_cost += cache_creation_tokens_5m * cache_creation_cost if cache_creation_tokens_5m is not None else 0.0
|
||||
total_cost += (
|
||||
cache_creation_tokens_1h * cache_creation_cost_above_1hr if cache_creation_tokens_1h is not None else 0.0
|
||||
)
|
||||
else:
|
||||
total_cost += cache_creation_tokens * cache_creation_cost
|
||||
return total_cost
|
||||
if cache_creation_token_details is None:
|
||||
return cache_creation_tokens * cache_creation_cost
|
||||
tokens_by_ttl: Final[Mapping[str, int | None]] = MappingProxyType(
|
||||
{
|
||||
"5m": cache_creation_token_details.ephemeral_5m_input_tokens,
|
||||
"30m": cache_creation_token_details.ephemeral_30m_input_tokens,
|
||||
"1hr": cache_creation_token_details.ephemeral_1h_input_tokens,
|
||||
"12hr": cache_creation_token_details.ephemeral_12h_input_tokens,
|
||||
}
|
||||
)
|
||||
unattributed_tokens: Final = max(cache_creation_tokens - sum(t or 0 for t in tokens_by_ttl.values()), 0)
|
||||
return unattributed_tokens * cache_creation_cost + sum(
|
||||
(tokens or 0) * cache_creation_cost_by_ttl.get(ttl, cache_creation_cost)
|
||||
for ttl, tokens in tokens_by_ttl.items()
|
||||
)
|
||||
|
||||
|
||||
class PromptTokensDetailsResult(TypedDict):
|
||||
|
|
@ -1178,7 +1201,7 @@ def _calculate_input_cost(
|
|||
prompt_base_cost: float,
|
||||
cache_read_cost: float,
|
||||
cache_creation_cost: float,
|
||||
cache_creation_cost_above_1hr: float,
|
||||
cache_creation_cost_by_ttl: Mapping[str, float],
|
||||
service_tier: str | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
|
|
@ -1248,8 +1271,8 @@ def _calculate_input_cost(
|
|||
prompt_cost += calculate_cache_writing_cost(
|
||||
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
|
||||
cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"],
|
||||
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
cache_creation_cost_by_ttl=cache_creation_cost_by_ttl,
|
||||
)
|
||||
|
||||
### CHARACTER COST
|
||||
|
|
@ -1510,34 +1533,31 @@ def generic_cost_per_token(
|
|||
)
|
||||
|
||||
billing_time: Final = current_time if current_time is not None else current_billing_time()
|
||||
(
|
||||
prompt_base_cost,
|
||||
completion_base_cost,
|
||||
cache_creation_cost,
|
||||
cache_creation_cost_above_1hr,
|
||||
cache_read_cost,
|
||||
) = _get_token_base_cost(
|
||||
base_costs: Final = _get_token_base_cost(
|
||||
model_info=resolved_model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
current_time=billing_time,
|
||||
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
|
||||
)
|
||||
completion_base_cost: Final = base_costs.output_rate
|
||||
resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate(
|
||||
model_info=resolved_model_info,
|
||||
usage=usage,
|
||||
current_time=billing_time,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
effective_cache_read_cost: Final = cache_read_cost if resolved_cache_read_cost is None else resolved_cache_read_cost
|
||||
effective_cache_read_cost: Final = (
|
||||
base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost
|
||||
)
|
||||
|
||||
prompt_cost = _calculate_input_cost(
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
model_info=resolved_model_info,
|
||||
prompt_base_cost=prompt_base_cost,
|
||||
prompt_base_cost=base_costs.input_rate,
|
||||
cache_read_cost=effective_cache_read_cost,
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
|
||||
cache_creation_cost=base_costs.cache_creation_rate,
|
||||
cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
|
|
@ -1640,7 +1660,7 @@ class BilledTokenRates:
|
|||
cache_read_input_token_cost: float
|
||||
cache_read_input_audio_token_cost: float
|
||||
cache_creation_input_token_cost: float
|
||||
cache_creation_input_token_cost_above_1hr: float
|
||||
cache_creation_input_token_cost_by_ttl: Mapping[str, float]
|
||||
output_cost_per_reasoning_token: float
|
||||
|
||||
def scaled(self, multiplier: float) -> "BilledTokenRates":
|
||||
|
|
@ -1652,7 +1672,9 @@ class BilledTokenRates:
|
|||
cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier,
|
||||
cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier,
|
||||
cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier,
|
||||
cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier,
|
||||
cache_creation_input_token_cost_by_ttl=MappingProxyType(
|
||||
{ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()}
|
||||
),
|
||||
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
|
||||
)
|
||||
|
||||
|
|
@ -1702,7 +1724,7 @@ def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRat
|
|||
cache_read_input_token_cost=cache_read_rate,
|
||||
cache_read_input_audio_token_cost=cache_read_rate,
|
||||
cache_creation_input_token_cost=cache_creation_rate,
|
||||
cache_creation_input_token_cost_above_1hr=cache_creation_rate,
|
||||
cache_creation_input_token_cost_by_ttl=MappingProxyType({}),
|
||||
output_cost_per_reasoning_token=output_rate,
|
||||
)
|
||||
|
||||
|
|
@ -1717,13 +1739,7 @@ def _cost_map_billed_rates(
|
|||
current_time: datetime | None,
|
||||
) -> BilledTokenRates:
|
||||
billing_time: Final = current_time if current_time is not None else current_billing_time()
|
||||
(
|
||||
prompt_base_cost,
|
||||
completion_base_cost,
|
||||
cache_creation_cost_rate,
|
||||
cache_creation_cost_above_1hr_rate,
|
||||
cache_read_cost_rate,
|
||||
) = _get_token_base_cost(
|
||||
base_costs: Final = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
|
|
@ -1737,14 +1753,14 @@ def _cost_map_billed_rates(
|
|||
service_tier=service_tier,
|
||||
)
|
||||
effective_cache_read_cost_rate: Final = (
|
||||
cache_read_cost_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
|
||||
base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate
|
||||
)
|
||||
|
||||
reasoning_rate: Final = _resolve_billed_reasoning_rate(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
completion_base_cost=completion_base_cost,
|
||||
completion_base_cost=base_costs.output_rate,
|
||||
current_time=billing_time,
|
||||
)
|
||||
audio_cache_read_rate: Final = get_cost_per_unit(
|
||||
|
|
@ -1759,14 +1775,14 @@ def _cost_map_billed_rates(
|
|||
* get_provider_specific_speed_multiplier(model_info=model_info, usage=usage)
|
||||
)
|
||||
return BilledTokenRates(
|
||||
input_cost_per_token=prompt_base_cost,
|
||||
output_cost_per_token=completion_base_cost,
|
||||
input_cost_per_token=base_costs.input_rate,
|
||||
output_cost_per_token=base_costs.output_rate,
|
||||
cache_read_input_token_cost=effective_cache_read_cost_rate,
|
||||
cache_read_input_audio_token_cost=(
|
||||
audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate
|
||||
),
|
||||
cache_creation_input_token_cost=cache_creation_cost_rate,
|
||||
cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
|
||||
cache_creation_input_token_cost=base_costs.cache_creation_rate,
|
||||
cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl,
|
||||
output_cost_per_reasoning_token=reasoning_rate,
|
||||
).scaled(multiplier)
|
||||
|
||||
|
|
@ -1854,8 +1870,8 @@ def get_token_type_cost_breakdown(
|
|||
else calculate_cache_writing_cost(
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=cache_creation_token_details,
|
||||
cache_creation_cost_above_1hr=rates.cache_creation_input_token_cost_above_1hr,
|
||||
cache_creation_cost=rates.cache_creation_input_token_cost,
|
||||
cache_creation_cost_by_ttl=rates.cache_creation_input_token_cost_by_ttl,
|
||||
)
|
||||
)
|
||||
return TokenTypeCostBreakdown(
|
||||
|
|
@ -1886,7 +1902,7 @@ def calculate_prompt_caching_savings(
|
|||
biller saw them rather than at the later spend write.
|
||||
"""
|
||||
model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider)
|
||||
prompt_base_cost, _, cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost = _get_token_base_cost(
|
||||
base_costs: Final = _get_token_base_cost(
|
||||
model_info=model_info_with_cache_read_default,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
|
|
@ -1899,9 +1915,11 @@ def calculate_prompt_caching_savings(
|
|||
current_time=billed_at,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
effective_cache_read_cost: Final = cache_read_cost if provider_cache_read_cost is None else provider_cache_read_cost
|
||||
write_rate: Final = cache_creation_cost or prompt_base_cost
|
||||
write_rate_1h: Final = cache_creation_cost_above_1hr or write_rate
|
||||
prompt_base_cost: Final = base_costs.input_rate
|
||||
effective_cache_read_cost: Final = (
|
||||
base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost
|
||||
)
|
||||
write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost
|
||||
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
|
||||
cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0)
|
||||
cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0)
|
||||
|
|
@ -1909,7 +1927,9 @@ def calculate_prompt_caching_savings(
|
|||
cache_creation_details: Final = (
|
||||
CacheCreationTokenDetails(
|
||||
ephemeral_5m_input_tokens=max(details.ephemeral_5m_input_tokens or 0, 0),
|
||||
ephemeral_30m_input_tokens=max(details.ephemeral_30m_input_tokens or 0, 0),
|
||||
ephemeral_1h_input_tokens=max(details.ephemeral_1h_input_tokens or 0, 0),
|
||||
ephemeral_12h_input_tokens=max(details.ephemeral_12h_input_tokens or 0, 0),
|
||||
)
|
||||
if details is not None
|
||||
else None
|
||||
|
|
@ -1918,8 +1938,10 @@ def calculate_prompt_caching_savings(
|
|||
write_premium: Final = calculate_cache_writing_cost(
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=cache_creation_details,
|
||||
cache_creation_cost_above_1hr=write_rate_1h - prompt_base_cost,
|
||||
cache_creation_cost=write_rate - prompt_base_cost,
|
||||
cache_creation_cost_by_ttl=MappingProxyType(
|
||||
{ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()}
|
||||
),
|
||||
)
|
||||
uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(
|
||||
model_info, vertex_location
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import Final
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
|
||||
from litellm.types.utils import (
|
||||
CompletionTokensDetailsWrapper,
|
||||
PromptTokensDetailsWrapper,
|
||||
|
|
@ -16,6 +17,8 @@ ZERO_COST_COUNTER_NAME: Final = "litellm_zero_cost_requests_total"
|
|||
|
||||
_TEXT_INPUT_RATE: Final = "input_cost_per_token"
|
||||
_AUDIO_INPUT_RATE: Final = "input_cost_per_audio_token"
|
||||
_CACHE_READ_INPUT_RATE: Final = "cache_read_input_token_cost"
|
||||
_CACHE_READ_AUDIO_INPUT_RATE: Final = "cache_read_input_audio_token_cost"
|
||||
_TEXT_OUTPUT_RATE: Final = "output_cost_per_token"
|
||||
_AUDIO_OUTPUT_RATE: Final = "output_cost_per_audio_token"
|
||||
_RATE_KEY_MARKERS: Final = ("cost", "pricing")
|
||||
|
|
@ -33,19 +36,38 @@ def _tokens(value: object) -> int:
|
|||
|
||||
|
||||
def used_pricing_keys(usage: Usage) -> tuple[str, ...]:
|
||||
prompt_audio: Final = _audio_tokens(usage.prompt_tokens_details)
|
||||
prompt_details: Final = parse_prompt_tokens_details(usage)
|
||||
prompt_audio: Final = prompt_details["audio_tokens"]
|
||||
completion_audio: Final = _audio_tokens(usage.completion_tokens_details)
|
||||
prompt_text: Final = _tokens(usage.prompt_tokens) - prompt_audio
|
||||
prompt_text: Final = max(
|
||||
_tokens(usage.prompt_tokens)
|
||||
- prompt_details["cache_hit_tokens"]
|
||||
- prompt_audio
|
||||
- prompt_details["image_tokens"]
|
||||
- prompt_details["video_tokens"],
|
||||
0,
|
||||
)
|
||||
cached_text: Final = max(prompt_details["cache_hit_tokens"] - prompt_details["cache_hit_audio_tokens"], 0)
|
||||
completion_text: Final = _tokens(usage.completion_tokens) - completion_audio
|
||||
components: Final = (
|
||||
(_TEXT_INPUT_RATE, prompt_text),
|
||||
(_AUDIO_INPUT_RATE, prompt_audio),
|
||||
(_CACHE_READ_INPUT_RATE, cached_text),
|
||||
(_CACHE_READ_AUDIO_INPUT_RATE, prompt_details["cache_hit_audio_tokens"]),
|
||||
(_TEXT_OUTPUT_RATE, completion_text),
|
||||
(_AUDIO_OUTPUT_RATE, completion_audio),
|
||||
)
|
||||
return tuple(key for key, count in components if count > 0)
|
||||
|
||||
|
||||
def _effective_pricing_key(key: str, pricing_entry: Mapping[str, object]) -> str:
|
||||
if key == _CACHE_READ_AUDIO_INPUT_RATE and pricing_entry.get(key) is None:
|
||||
return _CACHE_READ_INPUT_RATE if pricing_entry.get(_CACHE_READ_INPUT_RATE) is not None else _TEXT_INPUT_RATE
|
||||
if key == _CACHE_READ_INPUT_RATE and pricing_entry.get(key) is None:
|
||||
return _TEXT_INPUT_RATE
|
||||
return key
|
||||
|
||||
|
||||
def _nested_pricing(value: object) -> Mapping[str, object] | tuple[object, ...] | None:
|
||||
try:
|
||||
return _NESTED_PRICING.validate_python(value)
|
||||
|
|
@ -89,7 +111,8 @@ def diagnose_zero_cost(
|
|||
pricing_entry: Mapping[str, object],
|
||||
calculation_failed: bool,
|
||||
) -> StandardLoggingZeroCostDiagnostic | None:
|
||||
used_keys: Final = used_pricing_keys(usage)
|
||||
effective_keys: Final = tuple(_effective_pricing_key(key, pricing_entry) for key in used_pricing_keys(usage))
|
||||
used_keys: Final = tuple(key for index, key in enumerate(effective_keys) if key not in effective_keys[:index])
|
||||
if not used_keys:
|
||||
return None
|
||||
missing_keys: Final = tuple(key for key in used_keys if pricing_entry.get(key) is None)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate.
|
|||
See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
|
@ -75,7 +76,7 @@ def _flat_rates(model_info: ModelInfo) -> TokenRates:
|
|||
)
|
||||
|
||||
|
||||
def _tier_rates(model_info: ModelInfo, tier: dict) -> TokenRates:
|
||||
def _tier_rates(model_info: ModelInfo, tier: Mapping[str, object]) -> TokenRates:
|
||||
# A tier that declares output rates keeps the request on them, all-or-nothing. A tier table
|
||||
# spelling out only input rates would serve every completion for free, so there the model's
|
||||
# own output rates stand in
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_httpx_client,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AnthropicThinkingParam
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
|
|
@ -2364,26 +2365,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
elif modality == "DOCUMENT":
|
||||
cached_text_tokens = (cached_text_tokens or 0) + token_count
|
||||
|
||||
## Calculate non-cached tokens by subtracting cached from total (per modality)
|
||||
## This is necessary because promptTokensDetails includes both cached and non-cached tokens
|
||||
## See: https://github.com/BerriAI/litellm/issues/18750
|
||||
if cached_text_tokens is not None and prompt_text_tokens is not None:
|
||||
# Explicit caching: subtract cached tokens per modality from cacheTokensDetails
|
||||
prompt_text_tokens = prompt_text_tokens - cached_text_tokens
|
||||
elif (
|
||||
if (
|
||||
cached_tokens is not None
|
||||
and prompt_text_tokens is not None
|
||||
and cached_text_tokens is None
|
||||
and "cacheTokensDetails" not in usage_metadata
|
||||
):
|
||||
# Implicit caching: only cachedContentTokenCount is provided (no cacheTokensDetails)
|
||||
# Subtract from text tokens since implicit caching is primarily for text content
|
||||
# See: https://github.com/BerriAI/litellm/issues/16341
|
||||
prompt_text_tokens = prompt_text_tokens - cached_tokens
|
||||
if cached_audio_tokens is not None and prompt_audio_tokens is not None:
|
||||
prompt_audio_tokens = prompt_audio_tokens - cached_audio_tokens
|
||||
if cached_image_tokens is not None and prompt_image_tokens is not None:
|
||||
prompt_image_tokens = prompt_image_tokens - cached_image_tokens
|
||||
if cached_video_tokens is not None and prompt_video_tokens is not None:
|
||||
prompt_video_tokens = prompt_video_tokens - cached_video_tokens
|
||||
|
||||
|
|
@ -2404,9 +2392,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
billable_text_tokens: Final = (
|
||||
prompt_text_tokens if prompt_text_tokens is None else prompt_text_tokens + billable_tool_use_prompt_tokens
|
||||
)
|
||||
cached_tokens_details_fields: Final = MappingProxyType(
|
||||
{
|
||||
key: token_count
|
||||
for key, token_count in (
|
||||
("text_tokens", cached_text_tokens),
|
||||
("audio_tokens", cached_audio_tokens),
|
||||
("image_tokens", cached_image_tokens),
|
||||
)
|
||||
if token_count is not None
|
||||
}
|
||||
)
|
||||
cached_tokens_details: Final = (
|
||||
CachedTokensDetails.model_validate(cached_tokens_details_fields) if cached_tokens_details_fields else None
|
||||
)
|
||||
|
||||
prompt_tokens_details = PromptTokensDetailsWrapper(
|
||||
cached_tokens=cached_tokens,
|
||||
cached_tokens_details=cached_tokens_details,
|
||||
audio_tokens=prompt_audio_tokens,
|
||||
text_tokens=billable_text_tokens,
|
||||
image_tokens=prompt_image_tokens,
|
||||
|
|
|
|||
|
|
@ -31137,11 +31137,15 @@
|
|||
},
|
||||
"gemini/gemini-2.5-flash": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_audio_token_cost_batches": 1e-07,
|
||||
"cache_read_input_audio_token_cost_flex": 1e-07,
|
||||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_read_input_token_cost_batches": 3e-08,
|
||||
"cache_read_input_token_cost_flex": 3e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_flex": 5e-07,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -31466,11 +31470,15 @@
|
|||
},
|
||||
"gemini/gemini-2.5-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 3e-08,
|
||||
"cache_read_input_audio_token_cost_batches": 3e-08,
|
||||
"cache_read_input_audio_token_cost_flex": 3e-08,
|
||||
"cache_read_input_audio_token_cost_priority": 5.4e-08,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 1e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_priority": 1.8e-08,
|
||||
"input_cost_per_audio_token": 3e-07,
|
||||
"input_cost_per_audio_token_flex": 1.5e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -31760,12 +31768,16 @@
|
|||
},
|
||||
"gemini/gemini-3.1-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"cache_read_input_audio_token_cost_batches": 2.5e-08,
|
||||
"cache_read_input_audio_token_cost_flex": 2.5e-08,
|
||||
"cache_read_input_audio_token_cost_priority": 9e-08,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_audio_token_flex": 2.5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"input_cost_per_token_batches": 1.25e-07,
|
||||
"input_cost_per_token_flex": 1.25e-07,
|
||||
|
|
@ -31886,10 +31898,14 @@
|
|||
},
|
||||
"gemini/gemini-3-flash-preview": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_audio_token_cost_batches": 1e-07,
|
||||
"cache_read_input_audio_token_cost_flex": 1e-07,
|
||||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost_batches": 5e-08,
|
||||
"cache_read_input_token_cost_flex": 5e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_flex": 5e-07,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -36795,6 +36811,12 @@
|
|||
},
|
||||
"gpt-6-astra": {
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_12hr": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 5e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
|
||||
|
|
@ -36880,6 +36902,12 @@
|
|||
},
|
||||
"gpt-6-sol": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
|
||||
|
|
@ -36957,6 +36985,12 @@
|
|||
},
|
||||
"gpt-6-luna": {
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_12hr": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 5e-07,
|
||||
"cache_creation_input_token_cost_decisions": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
|
||||
|
|
@ -37043,6 +37077,12 @@
|
|||
},
|
||||
"gpt-5.6": {
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
|
||||
|
|
@ -37117,6 +37157,12 @@
|
|||
},
|
||||
"gpt-5.6-sol": {
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
|
||||
|
|
@ -37193,6 +37239,12 @@
|
|||
},
|
||||
"gpt-5.6-terra": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
|
||||
|
|
@ -37268,6 +37320,12 @@
|
|||
},
|
||||
"gpt-5.6-luna": {
|
||||
"cache_creation_input_token_cost": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
|
||||
|
|
@ -45847,6 +45905,7 @@
|
|||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
|
|
@ -45872,6 +45931,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
|
|
@ -49587,6 +49647,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"us-gov.openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -49606,6 +49667,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"us-gov.openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -60156,6 +60218,7 @@
|
|||
}
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"input_cost_per_token_batches": 7.5e-08,
|
||||
"input_cost_per_token_flex": 7.5e-08,
|
||||
|
|
@ -60181,6 +60244,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"input_cost_per_token_batches": 3.5e-08,
|
||||
"input_cost_per_token_flex": 3.5e-08,
|
||||
|
|
@ -60206,6 +60270,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"input_cost_per_token_batches": 7e-08,
|
||||
"input_cost_per_token_flex": 7e-08,
|
||||
|
|
@ -60229,6 +60294,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
|
|
@ -68080,6 +68146,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68099,6 +68166,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68377,6 +68445,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68396,6 +68465,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68784,6 +68854,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68803,6 +68874,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 7.2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68943,6 +69015,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68962,6 +69035,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 7.2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -74917,6 +74991,12 @@
|
|||
},
|
||||
"gpt-6.1-sol": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
|
|
|
|||
|
|
@ -22858,6 +22858,25 @@
|
|||
"title": "Body_image_edit_api_openai_deployments__model__images_edits_post",
|
||||
"type": "object"
|
||||
},
|
||||
"CacheWriteTokensDetails": {
|
||||
"additionalProperties": true,
|
||||
"properties": {
|
||||
"ttl_12h_tokens": {
|
||||
"title": "Ttl 12H Tokens",
|
||||
"type": "integer"
|
||||
},
|
||||
"ttl_30m_tokens": {
|
||||
"title": "Ttl 30M Tokens",
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"ttl_30m_tokens",
|
||||
"ttl_12h_tokens"
|
||||
],
|
||||
"title": "CacheWriteTokensDetails",
|
||||
"type": "object"
|
||||
},
|
||||
"CachedTokensDetails": {
|
||||
"properties": {
|
||||
"audio_tokens": {
|
||||
|
|
@ -24352,6 +24371,27 @@
|
|||
],
|
||||
"title": "Audio Tokens"
|
||||
},
|
||||
"cache_write_tokens": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Cache Write Tokens"
|
||||
},
|
||||
"cache_write_tokens_details": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/CacheWriteTokensDetails"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"cached_tokens": {
|
||||
"default": 0,
|
||||
"title": "Cached Tokens",
|
||||
|
|
|
|||
|
|
@ -3663,18 +3663,20 @@ class LiteLLMCompletionResponsesConfig:
|
|||
if count is not None
|
||||
}
|
||||
)
|
||||
response_usage.input_tokens_details = InputTokensDetails(
|
||||
cached_tokens=prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
|
||||
text_tokens=prompt_details.text_tokens,
|
||||
audio_tokens=prompt_details.audio_tokens,
|
||||
image_tokens=prompt_details.image_tokens,
|
||||
video_tokens=prompt_details.video_tokens,
|
||||
cached_tokens_details=(
|
||||
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
|
||||
),
|
||||
**cache_write_extra,
|
||||
**cache_type_extra,
|
||||
**grounding_request_counts,
|
||||
response_usage.input_tokens_details = InputTokensDetails.model_validate(
|
||||
{
|
||||
"cached_tokens": prompt_details.cached_tokens if prompt_details.cached_tokens is not None else 0,
|
||||
"text_tokens": prompt_details.text_tokens,
|
||||
"audio_tokens": prompt_details.audio_tokens,
|
||||
"image_tokens": prompt_details.image_tokens,
|
||||
"video_tokens": prompt_details.video_tokens,
|
||||
"cached_tokens_details": (
|
||||
cached_tokens_details if isinstance(cached_tokens_details, CachedTokensDetails) else None
|
||||
),
|
||||
**cache_write_extra,
|
||||
**cache_type_extra,
|
||||
**grounding_request_counts,
|
||||
}
|
||||
)
|
||||
|
||||
# Translate completion_tokens_details to output_tokens_details
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@ from contextlib import contextmanager
|
|||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
from typing_extensions import (
|
||||
TypeIs, # noqa: TID251 # narrows a client input item before its mark is read
|
||||
assert_never,
|
||||
|
|
@ -87,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
|
|||
from litellm.types.responses.main import *
|
||||
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params
|
||||
from litellm.types.utils import (
|
||||
ADDRESSED_RESPONSE_ID_FIELD,
|
||||
CallTypes,
|
||||
CostPerToken,
|
||||
CustomPricingLiteLLMParams,
|
||||
all_litellm_params,
|
||||
)
|
||||
from litellm.utils import (
|
||||
ProviderConfigManager,
|
||||
client,
|
||||
|
|
@ -547,6 +553,47 @@ async def aresponses_api_with_mcp(
|
|||
return response
|
||||
|
||||
|
||||
def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]:
|
||||
return MappingProxyType(
|
||||
litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True)
|
||||
)
|
||||
|
||||
|
||||
class _DeploymentModelInfo(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
id: object = None
|
||||
|
||||
|
||||
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
|
||||
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
|
||||
)
|
||||
|
||||
|
||||
def _is_router_deployment(deployment_model_info: object) -> bool:
|
||||
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
|
||||
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
|
||||
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
|
||||
|
||||
|
||||
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
|
||||
|
||||
|
||||
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
|
||||
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
|
||||
return None
|
||||
return _COST_PER_TOKEN.validate_python(
|
||||
litellm_params.model_dump(
|
||||
include={
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
},
|
||||
exclude_none=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _bridges_to_chat_completions(
|
||||
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
|
||||
) -> bool:
|
||||
|
|
@ -1674,6 +1721,7 @@ def responses(
|
|||
optional_params=dict(responses_api_request_params),
|
||||
litellm_params={
|
||||
**responses_api_request_params,
|
||||
**_per_call_custom_pricing(litellm_params),
|
||||
"aresponses": _is_async,
|
||||
"litellm_call_id": litellm_call_id,
|
||||
"model_info": kwargs.get("model_info"),
|
||||
|
|
@ -1684,6 +1732,9 @@ def responses(
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
if not _is_router_deployment(deployment_model_info):
|
||||
litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params)
|
||||
litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second
|
||||
# Decode any litellm-encoded encrypted-content item IDs back to their original IDs
|
||||
input = with_compaction_items_as_summaries(
|
||||
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
|
|||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
InputTokensDetails,
|
||||
OutputTokensDetails,
|
||||
ResponseAPIUsage,
|
||||
ResponseInputParam,
|
||||
|
|
@ -25,6 +26,7 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
from litellm.types.responses.main import DecodedResponseId
|
||||
from litellm.types.utils import (
|
||||
CacheCreationTokenDetails,
|
||||
CompletionTokensDetailsWrapper,
|
||||
PromptTokensDetailsWrapper,
|
||||
SpecialEnums,
|
||||
|
|
@ -47,6 +49,17 @@ def _output_token_detail(details: object, field: str) -> int | None:
|
|||
return value if isinstance(value, int) else None
|
||||
|
||||
|
||||
def _cache_write_lifetimes(cache_write_tokens_details: object) -> CacheCreationTokenDetails | None:
|
||||
"""OpenAI Responses splits `cache_write_tokens` into `ttl_30m_tokens` and `ttl_12h_tokens`."""
|
||||
ttl_30m_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_30m_tokens")
|
||||
ttl_12h_tokens: Final = _output_token_detail(cache_write_tokens_details, "ttl_12h_tokens")
|
||||
if ttl_30m_tokens is None and ttl_12h_tokens is None:
|
||||
return None
|
||||
return CacheCreationTokenDetails(
|
||||
ephemeral_30m_input_tokens=ttl_30m_tokens, ephemeral_12h_input_tokens=ttl_12h_tokens
|
||||
)
|
||||
|
||||
|
||||
def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a list is a Sequence of anything
|
||||
return isinstance(value, list)
|
||||
|
||||
|
|
@ -59,7 +72,11 @@ def _is_object_dict(
|
|||
|
||||
def _prompt_tokens_details_wrapper(details: object) -> PromptTokensDetailsWrapper:
|
||||
if _is_object_dict(details):
|
||||
return PromptTokensDetailsWrapper(**details)
|
||||
input_details: Final = InputTokensDetails.model_validate(details)
|
||||
return PromptTokensDetailsWrapper(
|
||||
**input_details.model_dump(exclude={"cache_write_tokens_details"}),
|
||||
cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details),
|
||||
)
|
||||
return PromptTokensDetailsWrapper(
|
||||
cached_tokens=getattr(details, "cached_tokens", None),
|
||||
audio_tokens=getattr(details, "audio_tokens", None),
|
||||
|
|
@ -68,6 +85,7 @@ def _prompt_tokens_details_wrapper(details: object) -> PromptTokensDetailsWrappe
|
|||
cached_tokens_details=getattr(details, "cached_tokens_details", None),
|
||||
video_tokens=getattr(details, "video_tokens", None),
|
||||
cache_write_tokens=getattr(details, "cache_write_tokens", None),
|
||||
cache_creation_token_details=_cache_write_lifetimes(getattr(details, "cache_write_tokens_details", None)),
|
||||
cache_type=getattr(details, "cache_type", None),
|
||||
web_search_requests=getattr(details, "web_search_requests", None),
|
||||
google_maps_grounding_requests=getattr(details, "google_maps_grounding_requests", None),
|
||||
|
|
|
|||
|
|
@ -14186,6 +14186,43 @@ class Router:
|
|||
deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _deployment_declares_no_vision(deployment: DeploymentTypedDict, group: str) -> bool:
|
||||
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
|
||||
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
|
||||
declared: Final = model_info.get("supports_vision")
|
||||
if declared is not None:
|
||||
return declared is False
|
||||
raw_params: Final = plain_deployment.get("litellm_params") or {}
|
||||
params: Final = cast(Mapping[str, object], raw_params) # cast-ok: router deployments are plain dicts
|
||||
if not params:
|
||||
return False
|
||||
base_model: Final = model_info.get("base_model") or params.get("base_model")
|
||||
try:
|
||||
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
|
||||
return litellm.utils.is_vision_explicitly_disabled(
|
||||
base_model if isinstance(base_model, str) else model, custom_llm_provider
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
|
||||
verbose_router_logger.debug(
|
||||
"litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e
|
||||
)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _request_carries_image(
|
||||
messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> bool:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
|
||||
|
||||
structured: Final = resolve_structured_messages(
|
||||
messages=cast(list[dict[str, object]] | None, messages), # cast-ok: chat messages are plain dicts
|
||||
request_kwargs=cast(dict[str, object], request_kwargs), # cast-ok: the resolver only reads the mapping
|
||||
)
|
||||
return bool(structured) and request_contains_image_content(structured)
|
||||
|
||||
@staticmethod
|
||||
def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]:
|
||||
requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs))
|
||||
|
|
@ -14212,6 +14249,7 @@ class Router:
|
|||
- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
|
||||
- filter models above rpm limits
|
||||
- if region given, filter out models not in that region / unknown region
|
||||
- request carries an image and the deployment declares supports_vision false, unless every candidate does
|
||||
- [TODO] function call and model doesn't support function calling
|
||||
"""
|
||||
|
||||
|
|
@ -14221,7 +14259,7 @@ class Router:
|
|||
# We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request)
|
||||
_returned_deployments = list(healthy_deployments)
|
||||
|
||||
invalid_model_indices: Final = set() # Use set for O(1) membership checks
|
||||
invalid_model_indices: Final = set[int]() # Use set for O(1) membership checks
|
||||
|
||||
_context_window_error = False
|
||||
_potential_error_str = ""
|
||||
|
|
@ -14371,9 +14409,20 @@ class Router:
|
|||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
if len(excluded_indices) > 0:
|
||||
remaining: Final = tuple(
|
||||
(idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices
|
||||
)
|
||||
text_only_indices: Final[frozenset[int]] = (
|
||||
frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model))
|
||||
if remaining and self._request_carries_image(messages, request_settings)
|
||||
else frozenset()
|
||||
)
|
||||
dropped_indices: Final[frozenset[int]] = excluded_indices | (
|
||||
text_only_indices if len(text_only_indices) < len(remaining) else frozenset()
|
||||
)
|
||||
if len(dropped_indices) > 0:
|
||||
# Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops)
|
||||
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices]
|
||||
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices]
|
||||
|
||||
return _returned_deployments
|
||||
|
||||
|
|
|
|||
|
|
@ -683,12 +683,12 @@ class ChatCompletionCachedContent(TypedDict):
|
|||
|
||||
class PromptCacheBreakpoint(TypedDict):
|
||||
mode: ReadOnly[Literal["explicit"]]
|
||||
ttl: NotRequired[ReadOnly[Literal["30m"]]]
|
||||
ttl: NotRequired[ReadOnly[Literal["30m", "12h"]]]
|
||||
|
||||
|
||||
class PromptCacheOptions(TypedDict, total=False):
|
||||
mode: ReadOnly[Literal["implicit", "explicit"]]
|
||||
ttl: ReadOnly[Literal["30m"]]
|
||||
ttl: ReadOnly[Literal["30m", "12h"]]
|
||||
|
||||
|
||||
class ChatCompletionThinkingBlock(TypedDict, total=False):
|
||||
|
|
@ -1382,10 +1382,23 @@ class OutputTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
|||
model_config = {"extra": "allow"}
|
||||
|
||||
|
||||
class CacheWriteTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
||||
ttl_30m_tokens: int
|
||||
ttl_12h_tokens: int
|
||||
|
||||
model_config = ConfigDict(extra="allow", frozen=True)
|
||||
|
||||
|
||||
def _is_none(value: object) -> bool:
|
||||
return value is None
|
||||
|
||||
|
||||
class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
||||
audio_tokens: int | None = None
|
||||
cached_tokens: int = 0
|
||||
cached_tokens_details: CachedTokensDetails | None = None
|
||||
cache_write_tokens: int | None = Field(default=None, exclude_if=_is_none)
|
||||
cache_write_tokens_details: CacheWriteTokensDetails | None = Field(default=None, exclude_if=_is_none)
|
||||
image_tokens: int | None = None
|
||||
text_tokens: int | None = None
|
||||
video_tokens: int | None = None
|
||||
|
|
|
|||
|
|
@ -322,11 +322,19 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
cache_creation_input_token_cost_above_272k_tokens_flex: float | None
|
||||
cache_creation_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_1hr: float | None
|
||||
cache_creation_input_token_cost_above_12hr: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_12hr_flex: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_12hr_priority: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: ReadOnly[float | None]
|
||||
cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing
|
||||
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
|
||||
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
|
||||
cache_read_input_token_cost: float | None
|
||||
cache_read_input_audio_token_cost: ReadOnly[float | None]
|
||||
cache_read_input_audio_token_cost_flex: ReadOnly[float | None]
|
||||
cache_read_input_audio_token_cost_priority: ReadOnly[float | None]
|
||||
cache_read_input_image_token_cost: ReadOnly[float | None]
|
||||
explicit_cache_read_input_token_cost: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
|
||||
|
|
@ -342,6 +350,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
cache_read_input_token_cost_above_272k_tokens_ultrafast: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_512k_tokens: float | None
|
||||
cache_read_input_token_cost_batches: ReadOnly[float | None]
|
||||
cache_read_input_audio_token_cost_batches: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None]
|
||||
|
|
@ -355,6 +364,8 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
off_peak_pricing: ReadOnly[OffPeakPricing | None] # time-windowed off-peak rates
|
||||
input_cost_per_character: float | None # only for vertex ai models
|
||||
input_cost_per_audio_token: float | None
|
||||
input_cost_per_audio_token_flex: ReadOnly[float | None]
|
||||
input_cost_per_audio_token_priority: ReadOnly[float | None]
|
||||
input_cost_per_token_above_128k_tokens: float | None # only for vertex ai models
|
||||
input_cost_per_token_above_200k_tokens: float | None # only for vertex ai gemini-2.5-pro models
|
||||
input_cost_per_token_above_100k_tokens: ReadOnly[float | None]
|
||||
|
|
@ -1804,7 +1815,9 @@ class CompletionTokensDetailsWrapper(CompletionTokensDetails): # wrapper for ol
|
|||
|
||||
class CacheCreationTokenDetails(LiteLLMBaseModel):
|
||||
ephemeral_5m_input_tokens: int | None = None
|
||||
ephemeral_30m_input_tokens: int | None = None
|
||||
ephemeral_1h_input_tokens: int | None = None
|
||||
ephemeral_12h_input_tokens: int | None = None
|
||||
|
||||
|
||||
class PromptTokensDetailsWrapper(
|
||||
|
|
@ -3954,6 +3967,12 @@ class MirroredPricingParams(LiteLLMBaseModel):
|
|||
explicit_cache_read_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost_above_1hr: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_flex: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_priority: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None
|
||||
tiered_pricing: list[dict[str, Any]] | None = None
|
||||
|
||||
|
||||
|
|
@ -4006,14 +4025,19 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
cache_read_input_token_cost_above_100k_tokens_batches: float | None = None
|
||||
cache_read_input_token_cost_above_200k_tokens_batches: float | None = None
|
||||
cache_read_input_token_cost_above_272k_tokens_batches: float | None = None
|
||||
cache_read_input_audio_token_cost_batches: float | None = None
|
||||
cache_creation_input_token_cost_batches: float | None = None
|
||||
cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None
|
||||
cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None
|
||||
cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None
|
||||
cache_read_input_audio_token_cost: float | None = None
|
||||
cache_read_input_audio_token_cost_flex: float | None = None
|
||||
cache_read_input_audio_token_cost_priority: float | None = None
|
||||
cache_read_input_image_token_cost: float | None = None
|
||||
input_cost_per_character_above_128k_tokens: float | None = None
|
||||
input_cost_per_audio_token: float | None = None
|
||||
input_cost_per_audio_token_flex: float | None = None
|
||||
input_cost_per_audio_token_priority: float | None = None
|
||||
input_cost_per_token_cache_hit: float | None = None
|
||||
input_cost_per_token_above_128k_tokens: float | None = None
|
||||
input_cost_per_token_above_100k_tokens: float | None = None
|
||||
|
|
|
|||
|
|
@ -6773,6 +6773,13 @@ def get_model_info_helper(
|
|||
),
|
||||
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
|
||||
cache_read_input_audio_token_cost=_model_info.get("cache_read_input_audio_token_cost", None),
|
||||
cache_read_input_audio_token_cost_flex=_model_info.get("cache_read_input_audio_token_cost_flex", None),
|
||||
cache_read_input_audio_token_cost_priority=_model_info.get(
|
||||
"cache_read_input_audio_token_cost_priority", None
|
||||
),
|
||||
cache_read_input_audio_token_cost_batches=_model_info.get(
|
||||
"cache_read_input_audio_token_cost_batches", None
|
||||
),
|
||||
explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None),
|
||||
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
|
||||
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
|
||||
|
|
@ -6817,6 +6824,24 @@ def get_model_info_helper(
|
|||
cache_creation_input_token_cost_above_1hr=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_1hr", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_flex=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_flex", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_priority=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_priority", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", None
|
||||
),
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority=_model_info.get(
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority", None
|
||||
),
|
||||
off_peak_pricing=_model_info.get("off_peak_pricing", None),
|
||||
input_cost_per_character=_model_info.get("input_cost_per_character", None),
|
||||
input_cost_per_token_above_128k_tokens=_model_info.get("input_cost_per_token_above_128k_tokens", None),
|
||||
|
|
@ -6840,6 +6865,8 @@ def get_model_info_helper(
|
|||
input_cost_per_second=_model_info.get("input_cost_per_second", None),
|
||||
input_cost_per_text_message=_model_info.get("input_cost_per_text_message", None),
|
||||
input_cost_per_audio_token=_model_info.get("input_cost_per_audio_token", None),
|
||||
input_cost_per_audio_token_flex=_model_info.get("input_cost_per_audio_token_flex", None),
|
||||
input_cost_per_audio_token_priority=_model_info.get("input_cost_per_audio_token_priority", None),
|
||||
input_cost_per_image_token=_model_info.get("input_cost_per_image_token", None),
|
||||
input_cost_per_video_token=_model_info.get("input_cost_per_video_token", None),
|
||||
input_cost_per_audio_token_batches=_model_info.get("input_cost_per_audio_token_batches", None),
|
||||
|
|
|
|||
|
|
@ -31137,11 +31137,15 @@
|
|||
},
|
||||
"gemini/gemini-2.5-flash": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_audio_token_cost_batches": 1e-07,
|
||||
"cache_read_input_audio_token_cost_flex": 1e-07,
|
||||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"cache_read_input_token_cost_batches": 3e-08,
|
||||
"cache_read_input_token_cost_flex": 3e-08,
|
||||
"cache_read_input_token_cost_priority": 5.4e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_flex": 5e-07,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -31466,11 +31470,15 @@
|
|||
},
|
||||
"gemini/gemini-2.5-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 3e-08,
|
||||
"cache_read_input_audio_token_cost_batches": 3e-08,
|
||||
"cache_read_input_audio_token_cost_flex": 3e-08,
|
||||
"cache_read_input_audio_token_cost_priority": 5.4e-08,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 1e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"cache_read_input_token_cost_priority": 1.8e-08,
|
||||
"input_cost_per_audio_token": 3e-07,
|
||||
"input_cost_per_audio_token_flex": 1.5e-07,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -31760,12 +31768,16 @@
|
|||
},
|
||||
"gemini/gemini-3.1-flash-lite": {
|
||||
"cache_read_input_audio_token_cost": 5e-08,
|
||||
"cache_read_input_audio_token_cost_batches": 2.5e-08,
|
||||
"cache_read_input_audio_token_cost_flex": 2.5e-08,
|
||||
"cache_read_input_audio_token_cost_priority": 9e-08,
|
||||
"cache_read_input_token_cost": 2.5e-08,
|
||||
"cache_read_input_token_cost_batches": 1.25e-08,
|
||||
"cache_read_input_token_cost_flex": 1.25e-08,
|
||||
"cache_read_input_token_cost_priority": 4.5e-08,
|
||||
"deprecation_date": "2027-05-07",
|
||||
"input_cost_per_audio_token": 5e-07,
|
||||
"input_cost_per_audio_token_flex": 2.5e-07,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"input_cost_per_token_batches": 1.25e-07,
|
||||
"input_cost_per_token_flex": 1.25e-07,
|
||||
|
|
@ -31886,10 +31898,14 @@
|
|||
},
|
||||
"gemini/gemini-3-flash-preview": {
|
||||
"cache_read_input_audio_token_cost": 1e-07,
|
||||
"cache_read_input_audio_token_cost_batches": 1e-07,
|
||||
"cache_read_input_audio_token_cost_flex": 1e-07,
|
||||
"cache_read_input_audio_token_cost_priority": 1.8e-07,
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"cache_read_input_token_cost_batches": 5e-08,
|
||||
"cache_read_input_token_cost_flex": 5e-08,
|
||||
"input_cost_per_audio_token": 1e-06,
|
||||
"input_cost_per_audio_token_flex": 5e-07,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "gemini",
|
||||
"max_input_tokens": 1048576,
|
||||
|
|
@ -36795,6 +36811,12 @@
|
|||
},
|
||||
"gpt-6-astra": {
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_12hr": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 0.0001,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 5e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
|
||||
|
|
@ -36880,6 +36902,12 @@
|
|||
},
|
||||
"gpt-6-sol": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
|
||||
|
|
@ -36957,6 +36985,12 @@
|
|||
},
|
||||
"gpt-6-luna": {
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_12hr": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 1e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 5e-07,
|
||||
"cache_creation_input_token_cost_decisions": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
|
||||
|
|
@ -37043,6 +37077,12 @@
|
|||
},
|
||||
"gpt-5.6": {
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
|
||||
|
|
@ -37117,6 +37157,12 @@
|
|||
},
|
||||
"gpt-5.6-sol": {
|
||||
"cache_creation_input_token_cost": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 4e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 2e-05,
|
||||
|
|
@ -37193,6 +37239,12 @@
|
|||
},
|
||||
"gpt-5.6-terra": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05,
|
||||
|
|
@ -37268,6 +37320,12 @@
|
|||
},
|
||||
"gpt-5.6-luna": {
|
||||
"cache_creation_input_token_cost": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-07,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
|
||||
|
|
@ -45847,6 +45905,7 @@
|
|||
"output_cost_per_token": 0.0
|
||||
},
|
||||
"openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
|
|
@ -45872,6 +45931,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions"
|
||||
],
|
||||
|
|
@ -49587,6 +49647,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"us-gov.openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -49606,6 +49667,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"us-gov.openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -60156,6 +60218,7 @@
|
|||
}
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"input_cost_per_token_batches": 7.5e-08,
|
||||
"input_cost_per_token_flex": 7.5e-08,
|
||||
|
|
@ -60181,6 +60244,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"input_cost_per_token_batches": 3.5e-08,
|
||||
"input_cost_per_token_flex": 3.5e-08,
|
||||
|
|
@ -60206,6 +60270,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"input_cost_per_token_batches": 7e-08,
|
||||
"input_cost_per_token_flex": 7e-08,
|
||||
|
|
@ -60229,6 +60294,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
|
|
@ -68080,6 +68146,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68099,6 +68166,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68377,6 +68445,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68396,6 +68465,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -68784,6 +68854,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68803,6 +68874,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 7.2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68943,6 +69015,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 8.4e-08,
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -68962,6 +69035,7 @@
|
|||
"supports_tool_choice": true
|
||||
},
|
||||
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
|
||||
"supports_vision": false,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 7.2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
|
|
@ -74917,6 +74991,12 @@
|
|||
},
|
||||
"gpt-6.1-sol": {
|
||||
"cache_creation_input_token_cost": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 1e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 2e-05,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 1e-05,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06,
|
||||
|
|
|
|||
|
|
@ -6985,6 +6985,106 @@ GPT_5_6_RESPONSES_CACHE_READ: Final = CostTrackingTestCase(
|
|||
),
|
||||
)
|
||||
|
||||
GPT_5_6_RESPONSES_CACHE_WRITE_12H: Final = CostTrackingTestCase(
|
||||
name="gpt-5.6-responses_cache_write_12h",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="gpt-5.6",
|
||||
endpoint="/v1/responses",
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"input": "summarize this text",
|
||||
"prompt_cache_options": {"mode": "implicit", "ttl": "12h"},
|
||||
},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "resp_$REQUEST_ID",
|
||||
"object": "response",
|
||||
"created_at": 1700000000,
|
||||
"status": "completed",
|
||||
"model": "gpt-5.6",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "scripted response", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 12928,
|
||||
"output_tokens": 380,
|
||||
"total_tokens": 13308,
|
||||
"input_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 5000,
|
||||
"cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000},
|
||||
},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
},
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.0344865,
|
||||
input_cost=0.0291865,
|
||||
output_cost=0.00532,
|
||||
prompt_tokens=12928,
|
||||
completion_tokens=380,
|
||||
cache_creation_cost=0.0153125,
|
||||
),
|
||||
)
|
||||
|
||||
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE: Final = CostTrackingTestCase(
|
||||
name="gpt-6-luna-responses_cache_write_12h_no_12h_rate",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="gpt-6-luna",
|
||||
endpoint="/v1/responses",
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"input": "summarize this text",
|
||||
"prompt_cache_options": {"mode": "implicit", "ttl": "12h"},
|
||||
},
|
||||
response=JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "resp_$REQUEST_ID",
|
||||
"object": "response",
|
||||
"created_at": 1700000000,
|
||||
"status": "completed",
|
||||
"model": "gpt-6-luna",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "scripted response", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": 12928,
|
||||
"output_tokens": 380,
|
||||
"total_tokens": 13308,
|
||||
"input_tokens_details": {
|
||||
"cached_tokens": 0,
|
||||
"cache_write_tokens": 5000,
|
||||
"cache_write_tokens_details": {"ttl_30m_tokens": 3000, "ttl_12h_tokens": 2000},
|
||||
},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
},
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.0016078,
|
||||
input_cost=0.0014178,
|
||||
output_cost=0.00019,
|
||||
prompt_tokens=12928,
|
||||
completion_tokens=380,
|
||||
cache_creation_cost=0.000625,
|
||||
),
|
||||
)
|
||||
|
||||
GPT_5_6_RESPONSES_REASONING: Final = CostTrackingTestCase(
|
||||
name="gpt-5.6-responses_reasoning",
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
|
|
@ -11913,6 +12013,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
OMNI_MODERATIONS_NEXT_SINGLE,
|
||||
OMNI_MODERATIONS_NEXT_LIST,
|
||||
GPT_5_6_RESPONSES_CACHE_READ,
|
||||
GPT_5_6_RESPONSES_CACHE_WRITE_12H,
|
||||
GPT_6_LUNA_RESPONSES_CACHE_WRITE_12H_NO_12H_RATE,
|
||||
GPT_5_6_RESPONSES_REASONING,
|
||||
GPT_5_6_RESPONSES_STREAM,
|
||||
GPT_5_6_RESPONSES_STREAM_CACHE_READ,
|
||||
|
|
|
|||
|
|
@ -174,6 +174,8 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
|
|||
},
|
||||
},
|
||||
"gpt-5.6": {
|
||||
"cache_creation_input_token_cost": 2.1875e-06,
|
||||
"cache_creation_input_token_cost_above_12hr": 4.375e-06,
|
||||
"cache_read_input_token_cost": 1.75e-07,
|
||||
"input_cost_per_audio_token": 4e-05,
|
||||
"input_cost_per_token": 1.75e-06,
|
||||
|
|
|
|||
|
|
@ -85,6 +85,12 @@ class CostMapEntry(BaseModel):
|
|||
cache_read_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost_above_1hr: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_flex: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_priority: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex: float | None = None
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority: float | None = None
|
||||
cache_creation_input_token_cost_above_1hr_above_200k_tokens: float | None = None
|
||||
cache_read_input_token_cost_above_200k_tokens: float | None = None
|
||||
cache_creation_input_token_cost_above_200k_tokens: float | None = None
|
||||
|
|
|
|||
|
|
@ -65,8 +65,8 @@ def test_missing_cache_read_rate_resolves_to_input_rate(prompt_tokens, read_rate
|
|||
prompt_cost, _ = generic_cost_per_token(
|
||||
"policy-fixture", usage, "openai", service_tier=service_tier, model_info=info
|
||||
)
|
||||
assert billed[4] == pytest.approx(read_rate if read_rate is not None else billed[0])
|
||||
assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed[0] + 100 * billed[4])
|
||||
assert billed.cache_read_rate == pytest.approx(read_rate if read_rate is not None else billed.input_rate)
|
||||
assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed.input_rate + 100 * billed.cache_read_rate)
|
||||
|
||||
|
||||
def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None:
|
||||
|
|
@ -248,7 +248,7 @@ def test_missing_cache_read_uses_off_peak_input_rate():
|
|||
}
|
||||
when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc)
|
||||
billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when)
|
||||
assert billed[0] == billed[4] == 5e-6
|
||||
assert billed.input_rate == billed.cache_read_rate == 5e-6
|
||||
|
||||
|
||||
def test_reasoning_tokens_no_price_set(_local_model_cost_map):
|
||||
|
|
@ -324,8 +324,6 @@ def test_reasoning_tokens_gemini(_local_model_cost_map):
|
|||
)
|
||||
|
||||
|
||||
|
||||
|
||||
def test_image_tokens_with_custom_pricing():
|
||||
"""Test that image_tokens in completion are properly costed with output_cost_per_image_token."""
|
||||
from unittest.mock import patch
|
||||
|
|
@ -578,7 +576,7 @@ def test_get_token_base_cost_picks_highest_crossed_tier():
|
|||
}
|
||||
usage = Usage(prompt_tokens=150_000, completion_tokens=10, total_tokens=150_010)
|
||||
|
||||
prompt_base_cost = _get_token_base_cost(model_info, usage)[0]
|
||||
prompt_base_cost = _get_token_base_cost(model_info, usage).input_rate
|
||||
|
||||
assert prompt_base_cost == 9e-6
|
||||
|
||||
|
|
@ -779,12 +777,12 @@ def test_get_token_base_cost_weekend_only_off_peak_rate():
|
|||
saturday_peak_hours = _get_token_base_cost(
|
||||
model_info, usage, current_time=datetime(2026, 8, 29, 2, 0, tzinfo=timezone.utc)
|
||||
)
|
||||
assert saturday_peak_hours[:2] == (5e-7, 1e-6)
|
||||
assert (saturday_peak_hours.input_rate, saturday_peak_hours.output_rate) == (5e-7, 1e-6)
|
||||
|
||||
monday_same_hours = _get_token_base_cost(
|
||||
model_info, usage, current_time=datetime(2026, 8, 24, 2, 0, tzinfo=timezone.utc)
|
||||
)
|
||||
assert monday_same_hours[:2] == (1e-6, 2e-6)
|
||||
assert (monday_same_hours.input_rate, monday_same_hours.output_rate) == (1e-6, 2e-6)
|
||||
|
||||
|
||||
def test_get_token_base_cost_applies_off_peak_pricing():
|
||||
|
|
@ -810,14 +808,14 @@ def test_get_token_base_cost_applies_off_peak_pricing():
|
|||
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
|
||||
|
||||
off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc))
|
||||
assert off_peak[0] == 5e-7
|
||||
assert off_peak[1] == 1e-6
|
||||
assert off_peak[4] == 5e-8
|
||||
assert off_peak.input_rate == 5e-7
|
||||
assert off_peak.output_rate == 1e-6
|
||||
assert off_peak.cache_read_rate == 5e-8
|
||||
|
||||
peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc))
|
||||
assert peak[0] == 1e-6
|
||||
assert peak[1] == 2e-6
|
||||
assert peak[4] == 1e-7
|
||||
assert peak.input_rate == 1e-6
|
||||
assert peak.output_rate == 2e-6
|
||||
assert peak.cache_read_rate == 1e-7
|
||||
|
||||
|
||||
def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates():
|
||||
|
|
@ -843,8 +841,8 @@ def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates():
|
|||
},
|
||||
)
|
||||
result = _get_token_base_cost(model_info, usage, current_time=when)
|
||||
assert result[0] == 1e-6
|
||||
assert result[1] == 2e-6
|
||||
assert result.input_rate == 1e-6
|
||||
assert result.output_rate == 2e-6
|
||||
|
||||
|
||||
def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset():
|
||||
|
|
@ -864,8 +862,8 @@ def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset():
|
|||
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
|
||||
|
||||
result = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc))
|
||||
assert result[0] == 5e-7
|
||||
assert result[1] == 2e-6
|
||||
assert result.input_rate == 5e-7
|
||||
assert result.output_rate == 2e-6
|
||||
|
||||
|
||||
def test_get_token_base_cost_off_peak_wins_over_threshold():
|
||||
|
|
@ -891,12 +889,12 @@ def test_get_token_base_cost_off_peak_wins_over_threshold():
|
|||
usage = Usage(prompt_tokens=250000, completion_tokens=250000, total_tokens=500000)
|
||||
|
||||
off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc))
|
||||
assert off_peak[0] == 5e-7
|
||||
assert off_peak[1] == 1e-6
|
||||
assert off_peak.input_rate == 5e-7
|
||||
assert off_peak.output_rate == 1e-6
|
||||
|
||||
peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc))
|
||||
assert peak[0] == 3e-6
|
||||
assert peak[1] == 4e-6
|
||||
assert peak.input_rate == 3e-6
|
||||
assert peak.output_rate == 4e-6
|
||||
|
||||
|
||||
def test_get_model_info_propagates_off_peak_fields():
|
||||
|
|
@ -948,10 +946,10 @@ def test_get_token_base_cost_off_peak_wins_over_tiered_pricing():
|
|||
usage = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100)
|
||||
|
||||
inside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc))
|
||||
assert inside[:2] == (5e-7, 1e-6)
|
||||
assert (inside.input_rate, inside.output_rate) == (5e-7, 1e-6)
|
||||
|
||||
outside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc))
|
||||
assert outside[:2] == (3e-6, 6e-6)
|
||||
assert (outside.input_rate, outside.output_rate) == (3e-6, 6e-6)
|
||||
|
||||
|
||||
def _register_off_peak_reasoning_model(
|
||||
|
|
@ -1153,17 +1151,17 @@ def test_get_token_base_cost_off_peak_cache_creation_rate():
|
|||
inside_window = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)
|
||||
|
||||
inside = _get_token_base_cost(model_info, usage, current_time=inside_window)
|
||||
assert inside[2] == 5e-7
|
||||
assert inside[3] == 2e-6
|
||||
assert inside.cache_creation_rate == 5e-7
|
||||
assert inside.cache_creation_rates_by_ttl["1hr"] == 2e-6
|
||||
|
||||
outside = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc))
|
||||
assert outside[2] == 1.25e-6
|
||||
assert outside.cache_creation_rate == 1.25e-6
|
||||
|
||||
without_key = cast(
|
||||
ModelInfo,
|
||||
{**model_info, "off_peak_pricing": {"hours_utc": "16:30-00:30", "input_cost_per_token": 5e-7}},
|
||||
)
|
||||
assert _get_token_base_cost(without_key, usage, current_time=inside_window)[2] == 1.25e-6
|
||||
assert _get_token_base_cost(without_key, usage, current_time=inside_window).cache_creation_rate == 1.25e-6
|
||||
|
||||
|
||||
def test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_creation_rates():
|
||||
|
|
@ -1583,19 +1581,14 @@ def test_generic_cost_per_token_tier_1hr_cache_rate_preserves_zero_and_fallback(
|
|||
),
|
||||
)
|
||||
costs: Final = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openrouter")
|
||||
breakdown: Final = get_token_type_cost_breakdown(
|
||||
model=model, usage=usage, custom_llm_provider="openrouter"
|
||||
)
|
||||
breakdown: Final = get_token_type_cost_breakdown(model=model, usage=usage, custom_llm_provider="openrouter")
|
||||
|
||||
ordinary_rate: Final = cache_rates.get("cache_creation_input_token_cost")
|
||||
expected_write_cost: Final = (
|
||||
(20 - one_hour_tokens) * (ordinary_rate if ordinary_rate is not None else 1.0)
|
||||
+ one_hour_tokens * expected_write_rate
|
||||
)
|
||||
expected_write_cost: Final = (20 - one_hour_tokens) * (
|
||||
ordinary_rate if ordinary_rate is not None else 1.0
|
||||
) + one_hour_tokens * expected_write_rate
|
||||
assert costs == pytest.approx((40 * 1.0 + 40 * 0.1 + expected_write_cost, 10 * 2.0))
|
||||
assert breakdown.cache_creation_cost == pytest.approx(expected_write_cost)
|
||||
assert breakdown.rates is not None
|
||||
assert breakdown.rates.cache_creation_input_token_cost_above_1hr == expected_write_rate
|
||||
finally:
|
||||
litellm.model_cost.pop(model, None)
|
||||
|
||||
|
|
@ -2020,8 +2013,8 @@ def test_calculate_cache_writing_cost():
|
|||
result = calculate_cache_writing_cost(
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=cache_creation_token_details,
|
||||
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr},
|
||||
)
|
||||
|
||||
# Expected calculation:
|
||||
|
|
@ -2041,8 +2034,8 @@ def test_calculate_cache_writing_cost():
|
|||
result_no_details = calculate_cache_writing_cost(
|
||||
cache_creation_tokens=cache_creation_tokens_no_details,
|
||||
cache_creation_token_details=cache_creation_token_details_none,
|
||||
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
|
||||
cache_creation_cost=cache_creation_cost_fallback,
|
||||
cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr},
|
||||
)
|
||||
|
||||
# Expected calculation when no details: 1000 * 5e-06 = 0.005
|
||||
|
|
@ -2059,15 +2052,13 @@ def test_calculate_cache_writing_cost():
|
|||
result_partial = calculate_cache_writing_cost(
|
||||
cache_creation_tokens=500,
|
||||
cache_creation_token_details=cache_creation_token_details_partial,
|
||||
cache_creation_cost_above_1hr=6e-06,
|
||||
cache_creation_cost=3e-06,
|
||||
cache_creation_cost_by_ttl={"1hr": 6e-06, "12hr": 6e-06},
|
||||
)
|
||||
|
||||
# Expected calculation: 0 (for None 5m tokens) + (100 * 6e-06) = 0.0006
|
||||
expected_cost_partial = (0.0) + (100 * 6e-06)
|
||||
expected_cost_partial = (400 * 3e-06) + (100 * 6e-06)
|
||||
|
||||
assert round(result_partial, 6) == round(expected_cost_partial, 6)
|
||||
assert round(result_partial, 6) == 0.0006
|
||||
|
||||
# Test case 4: Zero costs
|
||||
result_zero = calculate_cache_writing_cost(
|
||||
|
|
@ -2075,13 +2066,119 @@ def test_calculate_cache_writing_cost():
|
|||
cache_creation_token_details=CacheCreationTokenDetails(
|
||||
ephemeral_5m_input_tokens=50, ephemeral_1h_input_tokens=950
|
||||
),
|
||||
cache_creation_cost_above_1hr=0.0,
|
||||
cache_creation_cost=0.0,
|
||||
cache_creation_cost_by_ttl={"1hr": 0.0, "12hr": 0.0},
|
||||
)
|
||||
|
||||
assert result_zero == 0.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", CacheCreationTokenDetails.model_fields)
|
||||
def test_calculate_cache_writing_cost_bills_every_reported_lifetime(field: str) -> None:
|
||||
cost = calculate_cache_writing_cost(
|
||||
cache_creation_tokens=10,
|
||||
cache_creation_token_details=CacheCreationTokenDetails(**{field: 10}),
|
||||
cache_creation_cost=2.0,
|
||||
cache_creation_cost_by_ttl={"1hr": 2.0, "12hr": 2.0},
|
||||
)
|
||||
assert cost == 20.0
|
||||
|
||||
|
||||
def test_calculate_cache_writing_cost_bills_writes_of_an_unknown_lifetime_at_the_base_rate() -> None:
|
||||
cost = calculate_cache_writing_cost(
|
||||
cache_creation_tokens=10,
|
||||
cache_creation_token_details=CacheCreationTokenDetails(
|
||||
ephemeral_30m_input_tokens=2, ephemeral_12h_input_tokens=3
|
||||
),
|
||||
cache_creation_cost=1.0,
|
||||
cache_creation_cost_by_ttl={"1hr": 4.0, "12hr": 4.0},
|
||||
)
|
||||
assert cost == 2 * 1.0 + 3 * 4.0 + 5 * 1.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt_tokens", "service_tier", "write_rate", "twelve_hour_rate"),
|
||||
[
|
||||
(10_000, "default", 2e-6, 4e-6),
|
||||
(10_000, "flex", 1e-6, 2e-6),
|
||||
(10_000, "priority", 3e-6, 6e-6),
|
||||
(300_000, "default", 4e-6, 8e-6),
|
||||
(300_000, "flex", 2e-6, 4e-6),
|
||||
(300_000, "priority", 6e-6, 12e-6),
|
||||
],
|
||||
)
|
||||
def test_generic_cost_per_token_prices_12h_cache_writes_at_prompt_threshold(
|
||||
prompt_tokens: int, service_tier: str, write_rate: float, twelve_hour_rate: float
|
||||
) -> None:
|
||||
model_info: Final[ModelInfo] = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_creation_input_token_cost": 2e-6,
|
||||
"cache_creation_input_token_cost_flex": 1e-6,
|
||||
"cache_creation_input_token_cost_priority": 3e-6,
|
||||
"cache_creation_input_token_cost_above_272k_tokens": 4e-6,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_flex": 2e-6,
|
||||
"cache_creation_input_token_cost_above_272k_tokens_priority": 6e-6,
|
||||
"cache_creation_input_token_cost_above_12hr": 4e-6,
|
||||
"cache_creation_input_token_cost_above_12hr_flex": 2e-6,
|
||||
"cache_creation_input_token_cost_above_12hr_priority": 6e-6,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": 8e-6,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": 4e-6,
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": 12e-6,
|
||||
}
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=0,
|
||||
total_tokens=prompt_tokens,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cache_creation_tokens=5_000,
|
||||
cache_creation_token_details=CacheCreationTokenDetails(
|
||||
ephemeral_30m_input_tokens=3_000, ephemeral_12h_input_tokens=2_000
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
result: Final = generic_cost_per_token(
|
||||
model="test-12h-cache-write",
|
||||
usage=usage,
|
||||
custom_llm_provider="openai",
|
||||
service_tier=None if service_tier == "default" else service_tier,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert result[0] == pytest.approx(
|
||||
(usage.prompt_tokens - usage.prompt_tokens_details.cache_creation_tokens) * model_info["input_cost_per_token"]
|
||||
+ 3_000 * write_rate
|
||||
+ 2_000 * twelve_hour_rate
|
||||
)
|
||||
|
||||
|
||||
def test_generic_cost_per_token_falls_back_to_default_for_missing_12h_rate() -> None:
|
||||
model_info: Final[ModelInfo] = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_creation_input_token_cost": 3e-6,
|
||||
}
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=0,
|
||||
total_tokens=100,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cache_creation_tokens=100,
|
||||
cache_creation_token_details=CacheCreationTokenDetails(ephemeral_12h_input_tokens=100),
|
||||
),
|
||||
)
|
||||
|
||||
result: Final = generic_cost_per_token(
|
||||
model="test-12h-cache-write-fallback",
|
||||
usage=usage,
|
||||
custom_llm_provider="openai",
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert result[0] == pytest.approx(100 * model_info["cache_creation_input_token_cost"])
|
||||
|
||||
|
||||
def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
|
||||
"""
|
||||
Regression test: when cache_creation_tokens is 0 but cache_creation_token_details
|
||||
|
|
@ -2119,7 +2216,7 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details():
|
|||
prompt_base_cost=0.0,
|
||||
cache_read_cost=0.0,
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
|
||||
cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr},
|
||||
)
|
||||
|
||||
# Expected: (100 * 3.75e-06) + (200 * 6e-06) = 0.000375 + 0.0012 = 0.001575
|
||||
|
|
@ -2330,10 +2427,6 @@ def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(_local_mo
|
|||
assert round(cost, 10) == round(expected_cost, 10)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def test_query_count_is_free_without_a_per_query_price(_local_model_cost_map):
|
||||
usage = Usage(
|
||||
prompt_tokens=0,
|
||||
|
|
@ -2512,8 +2605,6 @@ def test_vertex_global_or_absent_location_no_uplift(vertex_location, _local_mode
|
|||
assert base == located
|
||||
|
||||
|
||||
|
||||
|
||||
def test_vertex_uplift_invalid_multiplier_defaults_to_one():
|
||||
"""A malformed multiplier in the cost map degrades to base pricing, never raises."""
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
|
|
@ -2569,10 +2660,10 @@ def test_flat_input_price_keeps_other_token_thresholds(field: str) -> None:
|
|||
model_info=info,
|
||||
usage=Usage(prompt_tokens=300_001, completion_tokens=3, total_tokens=300_004),
|
||||
)
|
||||
assert rates[0] == 0.001
|
||||
assert rates[1] == (0.005 if field == "output_cost_per_token" else 0.002)
|
||||
assert rates[2] == (0.005 if field == "cache_creation_input_token_cost" else 0.004)
|
||||
assert rates[4] == (0.005 if field == "cache_read_input_token_cost" else 0.003)
|
||||
assert rates.input_rate == 0.001
|
||||
assert rates.output_rate == (0.005 if field == "output_cost_per_token" else 0.002)
|
||||
assert rates.cache_creation_rate == (0.005 if field == "cache_creation_input_token_cost" else 0.004)
|
||||
assert rates.cache_read_rate == (0.005 if field == "cache_read_input_token_cost" else 0.003)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt_tokens", (200_000, 200_001, 512_000, 512_001))
|
||||
|
|
@ -2599,10 +2690,12 @@ def test_mixed_token_thresholds_preserve_each_crossed_rate(prompt_tokens: int, h
|
|||
model_info=info,
|
||||
usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=3, total_tokens=prompt_tokens + 3),
|
||||
)
|
||||
assert rates[0] == (0.005 if prompt_tokens > 200_000 else 0.001)
|
||||
assert rates[1] == (higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002)
|
||||
assert rates[2] == (0.007 if prompt_tokens > 200_000 else 0.004)
|
||||
assert rates[4] == (higher_rate if prompt_tokens > 512_000 else 0.003)
|
||||
assert rates.input_rate == (0.005 if prompt_tokens > 200_000 else 0.001)
|
||||
assert rates.output_rate == (
|
||||
higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002
|
||||
)
|
||||
assert rates.cache_creation_rate == (0.007 if prompt_tokens > 200_000 else 0.004)
|
||||
assert rates.cache_read_rate == (higher_rate if prompt_tokens > 512_000 else 0.003)
|
||||
|
||||
|
||||
def test_threshold_keys_exclude_service_tier_variants():
|
||||
|
|
@ -2626,8 +2719,7 @@ def test_threshold_keys_exclude_service_tier_variants():
|
|||
},
|
||||
)
|
||||
usage = Usage(prompt_tokens=350_000, completion_tokens=1_000, total_tokens=351_000)
|
||||
prompt_base, *_ = _get_token_base_cost(model_info=model_info, usage=usage)
|
||||
assert prompt_base == 3e-6
|
||||
assert _get_token_base_cost(model_info=model_info, usage=usage).input_rate == 3e-6
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -2897,7 +2989,7 @@ def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeyp
|
|||
cache_read_input_token_cost=6e-7,
|
||||
cache_read_input_audio_token_cost=6e-7,
|
||||
cache_creation_input_token_cost=7.5e-6,
|
||||
cache_creation_input_token_cost_above_1hr=7.5e-6,
|
||||
cache_creation_input_token_cost_by_ttl={"1hr": 7.5e-6, "12hr": 7.5e-6},
|
||||
output_cost_per_reasoning_token=3e-5,
|
||||
)
|
||||
assert breakdown.cache_read_cost == pytest.approx(200_000 * rates.cache_read_input_token_cost)
|
||||
|
|
@ -3713,6 +3805,23 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a
|
|||
3e-7,
|
||||
id="no-1h-price-uses-the-off-peak-write-price",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"tiered_pricing": [
|
||||
{
|
||||
"range": [0, 128000],
|
||||
"input_cost_per_token": 2e-7,
|
||||
"output_cost_per_token": 1.25e-6,
|
||||
"cache_creation_input_token_cost": 2.5e-7,
|
||||
}
|
||||
],
|
||||
"off_peak_pricing": {"hours_utc": "00:00-23:59", "cache_creation_input_token_cost": 3e-7},
|
||||
},
|
||||
datetime(2026, 9, 14, 12, tzinfo=timezone.utc),
|
||||
3e-7,
|
||||
3e-7,
|
||||
id="tiered-no-1h-price-uses-the-off-peak-write-price",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_path(
|
||||
|
|
@ -3724,10 +3833,17 @@ def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_
|
|||
model_info = {"input_cost_per_token": 2e-7, "output_cost_per_token": 1.25e-6, **cache_rates}
|
||||
usage = Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11)
|
||||
|
||||
_, _, creation, creation_1h, _ = _get_token_base_cost(model_info, usage, current_time=current_time)
|
||||
rates = _get_token_base_cost(model_info, usage, current_time=current_time)
|
||||
|
||||
assert creation == pytest.approx(expected_creation)
|
||||
assert creation_1h == pytest.approx(expected_creation_1h)
|
||||
assert rates.cache_creation_rate == pytest.approx(expected_creation)
|
||||
assert calculate_cache_writing_cost(
|
||||
cache_creation_tokens=3,
|
||||
cache_creation_token_details=CacheCreationTokenDetails(
|
||||
ephemeral_1h_input_tokens=1, ephemeral_12h_input_tokens=2
|
||||
),
|
||||
cache_creation_cost=rates.cache_creation_rate,
|
||||
cache_creation_cost_by_ttl=rates.cache_creation_rates_by_ttl,
|
||||
) == pytest.approx(3 * expected_creation_1h)
|
||||
|
||||
|
||||
def _image_response(num_images: int = 1, usage: ImageUsage | None = None) -> ImageResponse:
|
||||
|
|
@ -3787,8 +3903,6 @@ def test_route_image_generation_cost_openai_honors_deployment_input_cost_per_ima
|
|||
assert cost == pytest.approx(0.07)
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("custom_llm_provider", "model"),
|
||||
[
|
||||
|
|
@ -4206,17 +4320,28 @@ def test_token_breakdown_uses_supplied_pricing_without_catalog_lookup(rate: floa
|
|||
"cache_creation_input_token_cost": rate,
|
||||
}
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=20, completion_tokens=3, total_tokens=23,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5), cache_creation_input_tokens=5,
|
||||
prompt_tokens=20,
|
||||
completion_tokens=3,
|
||||
total_tokens=23,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=5),
|
||||
cache_creation_input_tokens=5,
|
||||
)
|
||||
prompt_cost, output_cost = generic_cost_per_token(
|
||||
model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", model_info=model_info,
|
||||
model="supplied-pricing-test",
|
||||
usage=usage,
|
||||
custom_llm_provider="bedrock",
|
||||
model_info=model_info,
|
||||
)
|
||||
breakdown: Final = get_token_type_cost_breakdown(
|
||||
model="supplied-pricing-test", usage=usage, custom_llm_provider="bedrock", model_info=model_info,
|
||||
model="supplied-pricing-test",
|
||||
usage=usage,
|
||||
custom_llm_provider="bedrock",
|
||||
model_info=model_info,
|
||||
)
|
||||
assert breakdown.cache_read_cost == pytest.approx(5 * rate)
|
||||
assert breakdown.cache_creation_cost == pytest.approx(5 * rate)
|
||||
assert breakdown.rates is not None
|
||||
assert prompt_cost == pytest.approx(10 * breakdown.rates.input_cost_per_token + breakdown.cache_read_cost + breakdown.cache_creation_cost)
|
||||
assert prompt_cost == pytest.approx(
|
||||
10 * breakdown.rates.input_cost_per_token + breakdown.cache_read_cost + breakdown.cache_creation_cost
|
||||
)
|
||||
assert output_cost == pytest.approx(3 * breakdown.rates.output_cost_per_token)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import respx
|
|||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
|
||||
|
||||
TIER_MODEL: Final = "tier-priced-test-model"
|
||||
|
|
@ -31,6 +32,20 @@ PROMPT_TOKENS: Final = 1000
|
|||
CACHED_TOKENS: Final = 200
|
||||
COMPLETION_TOKENS: Final = 500
|
||||
TIER_API_BASE: Final = "https://tier-pricing.invalid/v1"
|
||||
AUDIO_TIER_MODEL: Final = "tier-priced-gemini-audio-test-model"
|
||||
AUDIO_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
|
||||
{
|
||||
"input_cost_per_token": 4e-06,
|
||||
"output_cost_per_token": 8e-06,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
"input_cost_per_audio_token": 3e-06,
|
||||
"cache_read_input_audio_token_cost": 2e-06,
|
||||
"input_cost_per_audio_token_flex": 1e-06,
|
||||
"input_cost_per_audio_token_priority": 2e-06,
|
||||
"cache_read_input_audio_token_cost_flex": 4e-07,
|
||||
"cache_read_input_audio_token_cost_priority": 8e-07,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float:
|
||||
|
|
@ -78,6 +93,53 @@ def test_completion_cost_bills_the_price_columns_of_the_service_tier(
|
|||
assert cost == pytest.approx(_cost_at(TIER_ROW, column_suffix))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "audio_rate_key", "cache_audio_rate_key"),
|
||||
(
|
||||
pytest.param("flex", "input_cost_per_audio_token_flex", "cache_read_input_audio_token_cost_flex"),
|
||||
pytest.param("priority", "input_cost_per_audio_token_priority", "cache_read_input_audio_token_cost_priority"),
|
||||
),
|
||||
)
|
||||
def test_gemini_audio_and_cached_audio_use_service_tier_rates(
|
||||
local_model_cost_map: None, service_tier: str, audio_rate_key: str, cache_audio_rate_key: str
|
||||
) -> None:
|
||||
litellm.register_model(
|
||||
{
|
||||
AUDIO_TIER_MODEL: {
|
||||
"litellm_provider": "gemini",
|
||||
"mode": "chat",
|
||||
**AUDIO_TIER_ROW,
|
||||
}
|
||||
}
|
||||
)
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=40,
|
||||
completion_tokens=0,
|
||||
total_tokens=40,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=20,
|
||||
audio_tokens=20,
|
||||
cached_tokens=10,
|
||||
cached_tokens_details=CachedTokensDetails(audio_tokens=10),
|
||||
),
|
||||
)
|
||||
response: Final = ModelResponse(model=AUDIO_TIER_MODEL, usage=usage)
|
||||
|
||||
cost: Final = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=AUDIO_TIER_MODEL,
|
||||
custom_llm_provider="gemini",
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
expected: Final = (
|
||||
20 * AUDIO_TIER_ROW["input_cost_per_token"]
|
||||
+ 10 * AUDIO_TIER_ROW[audio_rate_key]
|
||||
+ 10 * AUDIO_TIER_ROW[cache_audio_rate_key]
|
||||
)
|
||||
assert cost == pytest.approx(expected)
|
||||
|
||||
|
||||
LONG_CONTEXT_TIER_MODEL: Final = "long-context-tier-priced-test-model"
|
||||
LONG_CONTEXT_TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import (
|
|||
used_pricing_keys,
|
||||
zero_cost_warning,
|
||||
)
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.utils import CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, Usage
|
||||
|
||||
PER_SECOND_ENTRY: Final = {"cost_per_second": 0.00042}
|
||||
|
|
@ -135,6 +136,36 @@ def test_audio_tokens_need_the_audio_rates() -> None:
|
|||
assert diagnostic["missing_pricing_keys"] == ("input_cost_per_audio_token", "output_cost_per_audio_token")
|
||||
|
||||
|
||||
def test_cached_audio_uses_its_cache_rate_in_zero_cost_diagnostic() -> None:
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=20,
|
||||
completion_tokens=0,
|
||||
total_tokens=20,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=0,
|
||||
audio_tokens=20,
|
||||
cached_tokens=20,
|
||||
cached_tokens_details=CachedTokensDetails(audio_tokens=20),
|
||||
),
|
||||
)
|
||||
pricing_entry: Final = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"input_cost_per_audio_token": 2e-6,
|
||||
"cache_read_input_audio_token_cost": 0,
|
||||
}
|
||||
|
||||
assert used_pricing_keys(usage) == ("cache_read_input_audio_token_cost",)
|
||||
assert (
|
||||
diagnose_zero_cost(
|
||||
usage=usage,
|
||||
pricing_model="gemini-audio",
|
||||
pricing_entry=pricing_entry,
|
||||
calculation_failed=False,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_warning_names_the_request_the_entry_the_missing_keys_and_the_counter() -> None:
|
||||
diagnostic = diagnose_zero_cost(
|
||||
usage=TEXT_USAGE, pricing_model="dep-1", pricing_entry=PER_SECOND_ENTRY, calculation_failed=False
|
||||
|
|
|
|||
|
|
@ -16,8 +16,10 @@ from pydantic import BaseModel
|
|||
import litellm
|
||||
from litellm import ModelResponse, completion, embedding, image_generation
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.cost_calculator import batch_cost_calculator
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, parse_prompt_tokens_details
|
||||
from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig
|
||||
|
|
@ -30,8 +32,9 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
|||
normalize_gemini_speech_config,
|
||||
reject_unknown_gemini_tts_voice_keys,
|
||||
)
|
||||
from litellm.types.llms.base import CachedTokensDetails
|
||||
from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata
|
||||
from litellm.types.utils import ChoiceLogprobs, Usage
|
||||
from litellm.types.utils import ChoiceLogprobs, ModelInfo, Usage
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map, CustomStreamWrapper
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -1515,6 +1518,7 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities():
|
|||
v = VertexGeminiConfig()
|
||||
usage_metadata = {
|
||||
"promptTokenCount": 210,
|
||||
"cachedContentTokenCount": 54,
|
||||
"candidatesTokenCount": 50,
|
||||
"totalTokenCount": 260,
|
||||
"promptTokensDetails": [
|
||||
|
|
@ -1536,9 +1540,10 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities():
|
|||
usage_metadata = UsageMetadata(**usage_metadata)
|
||||
result = v.calculate_usage(completion_response={"usageMetadata": usage_metadata})
|
||||
|
||||
# prompt details are total - cached per modality
|
||||
assert result.prompt_tokens_details.text_tokens == 16 # 20 - 4
|
||||
assert result.prompt_tokens_details.image_tokens == 140 # (90 + 100) - (40 + 10)
|
||||
assert result.prompt_tokens_details.text_tokens == 20
|
||||
assert result.prompt_tokens_details.image_tokens == 190
|
||||
assert result.prompt_tokens_details.cached_tokens == 54
|
||||
assert result.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(text_tokens=4, image_tokens=50)
|
||||
|
||||
# candidates details accumulate duplicate modalities
|
||||
assert result.completion_tokens_details.text_tokens == 20 # 15 + 5
|
||||
|
|
@ -1546,6 +1551,118 @@ def test_vertex_ai_usage_metadata_accumulates_duplicate_modalities():
|
|||
assert result.completion_tokens == 50
|
||||
|
||||
|
||||
def _gemini_usage_with_cached_audio(cached_audio_tokens: int) -> Usage:
|
||||
cache_tokens_details: Final = (
|
||||
(
|
||||
{"modality": "TEXT", "tokenCount": 5},
|
||||
{"modality": "AUDIO", "tokenCount": cached_audio_tokens},
|
||||
)
|
||||
if cached_audio_tokens > 0
|
||||
else ({"modality": "TEXT", "tokenCount": 5},)
|
||||
)
|
||||
completion_response: Final = {
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 40,
|
||||
"cachedContentTokenCount": 5 + cached_audio_tokens,
|
||||
"promptTokensDetails": (
|
||||
{"modality": "TEXT", "tokenCount": 20},
|
||||
{"modality": "AUDIO", "tokenCount": 20},
|
||||
),
|
||||
"cacheTokensDetails": cache_tokens_details,
|
||||
"candidatesTokenCount": 7,
|
||||
"totalTokenCount": 47,
|
||||
}
|
||||
}
|
||||
return VertexGeminiConfig.calculate_usage(completion_response=completion_response)
|
||||
|
||||
|
||||
def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_batch_rate() -> None:
|
||||
model_info: Final[ModelInfo] = {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"input_cost_per_token_batches": 1e-6,
|
||||
"input_cost_per_audio_token": 6e-6,
|
||||
"input_cost_per_audio_token_batches": 3e-6,
|
||||
"cache_read_input_token_cost": 8e-7,
|
||||
"cache_read_input_token_cost_batches": 4e-7,
|
||||
"cache_read_input_audio_token_cost": 5e-7,
|
||||
"cache_read_input_audio_token_cost_batches": 2e-7,
|
||||
}
|
||||
usage: Final = _gemini_usage_with_cached_audio(5)
|
||||
expected_prompt_cost: Final = (
|
||||
15 * model_info["input_cost_per_token_batches"]
|
||||
+ 15 * model_info["input_cost_per_audio_token_batches"]
|
||||
+ 5 * model_info["cache_read_input_token_cost_batches"]
|
||||
+ 5 * model_info["cache_read_input_audio_token_cost_batches"]
|
||||
)
|
||||
|
||||
prompt_cost, _ = batch_cost_calculator(
|
||||
usage=usage,
|
||||
model="gemini-audio-batch-test-model",
|
||||
custom_llm_provider="gemini",
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(expected_prompt_cost)
|
||||
|
||||
|
||||
def test_gemini_raw_usage_metadata_prices_cached_audio_at_its_standard_rate() -> None:
|
||||
model_info: Final[ModelInfo] = {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"input_cost_per_audio_token": 6e-6,
|
||||
"cache_read_input_token_cost": 8e-7,
|
||||
"cache_read_input_audio_token_cost": 5e-7,
|
||||
}
|
||||
usage: Final = _gemini_usage_with_cached_audio(5)
|
||||
expected_prompt_cost: Final = (
|
||||
15 * model_info["input_cost_per_token"]
|
||||
+ 15 * model_info["input_cost_per_audio_token"]
|
||||
+ 5 * model_info["cache_read_input_token_cost"]
|
||||
+ 5 * model_info["cache_read_input_audio_token_cost"]
|
||||
)
|
||||
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model="gemini-audio-standard-test-model",
|
||||
usage=usage,
|
||||
custom_llm_provider="gemini",
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(expected_prompt_cost)
|
||||
|
||||
|
||||
def test_gemini_usage_totals_are_subtracted_once_and_audio_cache_rate_matches_audio_rate() -> None:
|
||||
model_info: Final[ModelInfo] = {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"input_cost_per_audio_token": 6e-6,
|
||||
"cache_read_input_token_cost": 8e-7,
|
||||
"cache_read_input_audio_token_cost": 6e-6,
|
||||
}
|
||||
cached_usage: Final = _gemini_usage_with_cached_audio(5)
|
||||
uncached_usage: Final = _gemini_usage_with_cached_audio(0)
|
||||
cached_details: Final = parse_prompt_tokens_details(cached_usage)
|
||||
cached_prompt_cost, _ = generic_cost_per_token(
|
||||
model="gemini-audio-standard-test-model",
|
||||
usage=cached_usage,
|
||||
custom_llm_provider="gemini",
|
||||
model_info=model_info,
|
||||
)
|
||||
uncached_prompt_cost, _ = generic_cost_per_token(
|
||||
model="gemini-audio-standard-test-model",
|
||||
usage=uncached_usage,
|
||||
custom_llm_provider="gemini",
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert cached_usage.prompt_tokens_details.text_tokens == 20
|
||||
assert cached_usage.prompt_tokens_details.audio_tokens == 20
|
||||
assert cached_usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(
|
||||
text_tokens=5, audio_tokens=5
|
||||
)
|
||||
assert cached_details["text_tokens"] == 15
|
||||
assert cached_details["audio_tokens"] == 15
|
||||
assert cached_prompt_cost == pytest.approx(uncached_prompt_cost)
|
||||
|
||||
|
||||
def test_vertex_ai_map_thinking_param_with_budget_tokens_0():
|
||||
"""
|
||||
If budget_tokens is 0, do not set includeThoughts to True
|
||||
|
|
@ -6150,7 +6267,7 @@ def test_vertex_ai_usage_metadata_with_document_tokens_in_prompt():
|
|||
|
||||
|
||||
def test_vertex_ai_usage_metadata_with_document_tokens_cached():
|
||||
"""Test that cached DOCUMENT tokens are correctly subtracted from prompt text tokens."""
|
||||
"""Test cached DOCUMENT token details and unchanged prompt cost."""
|
||||
v = VertexGeminiConfig()
|
||||
|
||||
usage_metadata_dict = {
|
||||
|
|
@ -6174,10 +6291,24 @@ def test_vertex_ai_usage_metadata_with_document_tokens_cached():
|
|||
completion_response = {"usageMetadata": usage_metadata_dict}
|
||||
result = v.calculate_usage(completion_response=completion_response)
|
||||
|
||||
# DOCUMENT cached tokens map to cached_text_tokens, so:
|
||||
# text_tokens = (8 TEXT + 774 DOCUMENT) - 400 cached = 382
|
||||
assert result.prompt_tokens_details.text_tokens == 382, "text_tokens should be (8 + 774) - 400 cached = 382"
|
||||
assert result.prompt_tokens_details.text_tokens == 782
|
||||
assert result.prompt_tokens_details.cached_tokens == 400
|
||||
assert result.prompt_tokens_details.cached_tokens_details.text_tokens == 400
|
||||
|
||||
model_info: Final[ModelInfo] = {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 5e-7,
|
||||
}
|
||||
prompt_cost, _ = generic_cost_per_token(
|
||||
model="gemini-document-cache-test-model",
|
||||
usage=result,
|
||||
custom_llm_provider="gemini",
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(
|
||||
382 * model_info["input_cost_per_token"] + 400 * model_info["cache_read_input_token_cost"]
|
||||
)
|
||||
|
||||
|
||||
def test_async_streaming_uses_custom_client():
|
||||
|
|
|
|||
|
|
@ -3398,7 +3398,9 @@ class TestUsageTransformation:
|
|||
assert response_usage.input_tokens_details is not None
|
||||
assert response_usage.input_tokens_details.cached_tokens == 5
|
||||
assert response_usage.input_tokens_details.text_tokens == 8
|
||||
assert "cache_write_tokens" not in response_usage.input_tokens_details.model_dump()
|
||||
assert {"cache_write_tokens", "cache_write_tokens_details"}.isdisjoint(
|
||||
response_usage.input_tokens_details.model_dump()
|
||||
)
|
||||
|
||||
def test_transform_usage_with_cached_tokens_gemini(self):
|
||||
"""Test that cached_tokens from Gemini are properly transformed to input_tokens_details"""
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ import asyncio
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
import math
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from pathlib import Path
|
||||
from importlib import import_module
|
||||
from typing import Final
|
||||
|
|
@ -17,10 +18,12 @@ from unittest.mock import AsyncMock, patch
|
|||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from tests.unit.llms.bedrock.slow_upstream import (
|
||||
STREAM_TIMEOUT_SECONDS,
|
||||
UPSTREAM_FIRST_BYTE_SECONDS,
|
||||
|
|
@ -333,7 +336,9 @@ async def test_aresponses_sends_the_declared_reasoning_effort_level_rounded_up(
|
|||
|
||||
|
||||
def _effort_levels_disabled_by(entry: dict) -> tuple[str, ...]:
|
||||
return tuple(level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False)
|
||||
return tuple(
|
||||
level for level in ("none", "minimal", "low") if entry.get(f"supports_{level}_reasoning_effort") is False
|
||||
)
|
||||
|
||||
|
||||
def _gpt_5_row_disabling_a_level(provider: str) -> tuple[str, str]:
|
||||
|
|
@ -623,7 +628,8 @@ async def test_aresponses_websocket_strips_responses_routing_prefix_from_openai_
|
|||
from litellm.responses.main import _aresponses_websocket
|
||||
|
||||
with patch.object(
|
||||
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
|
||||
import_module("litellm.responses.main").base_llm_http_handler,
|
||||
"async_responses_websocket",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws:
|
||||
await _aresponses_websocket(
|
||||
|
|
@ -645,7 +651,8 @@ async def test_aresponses_websocket_keeps_routing_hints_out_of_the_relay_kwargs(
|
|||
from litellm.responses.main import _aresponses_websocket
|
||||
|
||||
with patch.object(
|
||||
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
|
||||
import_module("litellm.responses.main").base_llm_http_handler,
|
||||
"async_responses_websocket",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws:
|
||||
await _aresponses_websocket(
|
||||
|
|
@ -671,7 +678,9 @@ _ORIGINAL_WS_INPUT = [
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("nested", [False, True])
|
||||
async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame(nested: bool): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket
|
||||
async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame(
|
||||
nested: bool,
|
||||
): # test-quality-ok: the first frame handed to the relay is the only place the routed input is observable before the provider socket
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.responses.main import _aresponses_websocket
|
||||
|
|
@ -682,7 +691,8 @@ async def test_aresponses_websocket_forwards_the_routed_input_in_the_first_frame
|
|||
)
|
||||
|
||||
with patch.object(
|
||||
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
|
||||
import_module("litellm.responses.main").base_llm_http_handler,
|
||||
"async_responses_websocket",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws:
|
||||
await _aresponses_websocket(
|
||||
|
|
@ -711,7 +721,8 @@ async def test_aresponses_websocket_forwards_the_first_frame_verbatim_when_routi
|
|||
first_message = '{"type": "response.create", "model": "gpt-5.6", "input": [{"role": "user", "content": "hi"}]}'
|
||||
|
||||
with patch.object(
|
||||
import_module("litellm.responses.main").base_llm_http_handler, "async_responses_websocket",
|
||||
import_module("litellm.responses.main").base_llm_http_handler,
|
||||
"async_responses_websocket",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_ws:
|
||||
await _aresponses_websocket(
|
||||
|
|
@ -968,9 +979,7 @@ async def test_injection_points_reach_a_foundry_deployment_of_an_openai_model(mo
|
|||
Azure OpenAI one the Foundry deployment got Anthropic marks the Responses transform then stripped."""
|
||||
monkeypatch.setenv("AZURE_AI_API_BASE", "https://other-deployment.openai.azure.com")
|
||||
injected_client = AsyncHTTPHandler()
|
||||
mock_post = AsyncMock(
|
||||
return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200)
|
||||
)
|
||||
mock_post = AsyncMock(return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200))
|
||||
injected_client.post = mock_post
|
||||
|
||||
await litellm.aresponses(
|
||||
|
|
@ -1234,6 +1243,77 @@ def test_faked_stream_accepts_a_text_timeout(monkeypatch: pytest.MonkeyPatch) ->
|
|||
]
|
||||
|
||||
|
||||
async def _invoke_responses(sync_mode: bool, model: str, pricing: Mapping[str, float]) -> ResponsesAPIResponse:
|
||||
response: Final = (
|
||||
litellm.responses(model=model, input="hi", **pricing)
|
||||
if sync_mode
|
||||
else await litellm.aresponses(model=model, input="hi", **pricing)
|
||||
)
|
||||
assert isinstance(response, ResponsesAPIResponse)
|
||||
return response
|
||||
|
||||
|
||||
def _response_cost(response: ResponsesAPIResponse) -> float:
|
||||
hidden_params: Final[dict[str, float]] = getattr(response, "_hidden_params")
|
||||
return hidden_params["response_cost"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sync_mode", (True, False))
|
||||
@pytest.mark.parametrize("model", ("openai/gpt-5.4-nano", "openai/custom-priced-model-not-in-cost-map"))
|
||||
async def test_responses_bills_per_call_custom_pricing_from_usage(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, sync_mode: bool, model: str
|
||||
):
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost")
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(model_cost))
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_custom_price", model.split("/")[1]))
|
||||
)
|
||||
pricing: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5}
|
||||
|
||||
response: Final = await _invoke_responses(sync_mode, model, pricing)
|
||||
|
||||
assert response.usage is not None
|
||||
expected: Final = (
|
||||
response.usage.input_tokens * pricing["input_cost_per_token"]
|
||||
+ response.usage.output_tokens * pricing["output_cost_per_token"]
|
||||
)
|
||||
assert math.isclose(_response_cost(response), expected)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_responses_calls_keep_their_own_per_call_prices(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
):
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "fake-api-key")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
model: Final = "openai/gpt-5.4-nano"
|
||||
model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost")
|
||||
cost_map_row_before: Final = copy.deepcopy(model_cost.get(model))
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_concurrent", "gpt-5.4-nano"))
|
||||
)
|
||||
rates: Final = ((1e-5, 2e-5), (3e-5, 4e-5))
|
||||
|
||||
responses: Final = await asyncio.gather(
|
||||
*(
|
||||
_invoke_responses(False, model, {"input_cost_per_token": in_rate, "output_cost_per_token": out_rate})
|
||||
for in_rate, out_rate in rates
|
||||
)
|
||||
)
|
||||
|
||||
for response, (in_rate, out_rate) in zip(responses, rates, strict=True):
|
||||
assert response.usage is not None
|
||||
assert math.isclose(
|
||||
_response_cost(response), response.usage.input_tokens * in_rate + response.usage.output_tokens * out_rate
|
||||
)
|
||||
assert model_cost.get(model) == cost_map_row_before
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -1339,3 +1419,49 @@ async def test_responses_opencode_session_uses_effective_destination(
|
|||
assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123")
|
||||
assert sent.headers["authorization"] == "Bearer test-key"
|
||||
assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello"}
|
||||
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_responses_deployment_pricing_keeps_inherited_cache_rates(
|
||||
monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
|
||||
):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
model_cost: Final[dict[str, object]] = getattr(litellm, "model_cost")
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(model_cost))
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
rates: Final = {"input_cost_per_token": 1e-5, "output_cost_per_token": 2e-5}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "nano",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-nano", "api_key": "fake-api-key", **rates},
|
||||
"model_info": {"id": "nano-deployment"},
|
||||
}
|
||||
]
|
||||
)
|
||||
payload: Final[dict[str, object]] = _minimal_responses_api_payload("resp_router_cache", "gpt-5.4-nano")
|
||||
usage: Final = _JSON_OBJECT.validate_python(payload["usage"])
|
||||
cached_tokens: Final = 8
|
||||
respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={**payload, "usage": {**usage, "input_tokens_details": {"cached_tokens": cached_tokens}}},
|
||||
)
|
||||
)
|
||||
|
||||
response: Final = await router.aresponses(model="nano", input="hi")
|
||||
|
||||
assert isinstance(response, ResponsesAPIResponse)
|
||||
assert response.usage is not None
|
||||
uncached_input: Final = response.usage.input_tokens - cached_tokens
|
||||
full_price: Final = (
|
||||
response.usage.input_tokens * rates["input_cost_per_token"]
|
||||
+ response.usage.output_tokens * rates["output_cost_per_token"]
|
||||
)
|
||||
floor: Final = (
|
||||
uncached_input * rates["input_cost_per_token"] + response.usage.output_tokens * rates["output_cost_per_token"]
|
||||
)
|
||||
assert floor < _response_cost(response) < full_price
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from importlib import import_module
|
||||
import base64
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -7,10 +8,11 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
|
||||
|
|
@ -413,6 +415,43 @@ class TestResponseAPILoggingUtils:
|
|||
assert result.prompt_tokens_details.cache_creation_tokens == 10059
|
||||
assert result.prompt_tokens_details.cached_tokens == 0
|
||||
|
||||
@pytest.mark.parametrize("as_model", [False, True])
|
||||
def test_transform_response_api_usage_prices_12h_cache_writes(self, as_model: bool) -> None:
|
||||
usage_data: Final = {
|
||||
"input_tokens": 5_000,
|
||||
"output_tokens": 0,
|
||||
"total_tokens": 5_000,
|
||||
"input_tokens_details": {
|
||||
"cache_write_tokens": 5_000,
|
||||
"cache_write_tokens_details": {"ttl_30m_tokens": 3_000, "ttl_12h_tokens": 2_000},
|
||||
},
|
||||
}
|
||||
usage_input: Final = ResponseAPIUsage(**usage_data) if as_model else usage_data
|
||||
usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_input)
|
||||
model_info: Final[ModelInfo] = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_creation_input_token_cost": 3e-6,
|
||||
"cache_creation_input_token_cost_above_12hr": 5e-6,
|
||||
}
|
||||
|
||||
assert usage.prompt_tokens_details is not None
|
||||
assert usage.prompt_tokens_details.cache_write_tokens == 5_000
|
||||
assert usage.prompt_tokens_details.cache_creation_token_details is not None
|
||||
assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_12h_input_tokens == 2_000
|
||||
assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_30m_input_tokens == 3_000
|
||||
cost: Final = generic_cost_per_token(
|
||||
model="test-12h-cache-write",
|
||||
usage=usage,
|
||||
custom_llm_provider="openai",
|
||||
model_info=model_info,
|
||||
)[0]
|
||||
|
||||
assert cost == pytest.approx(
|
||||
3_000 * model_info["cache_creation_input_token_cost"]
|
||||
+ 2_000 * model_info["cache_creation_input_token_cost_above_12hr"]
|
||||
)
|
||||
|
||||
def test_transform_response_api_usage_mixed_details(self):
|
||||
"""Test transformation handles mixed token details (cached + image + audio)."""
|
||||
# Setup - hypothetical usage with mixed token types
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import litellm
|
|||
from litellm import TranscriptionResponse, model_cost
|
||||
from litellm.constants import REALTIME_METERED_USAGE_KEY
|
||||
from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
get_fallback_generalization_rules,
|
||||
|
|
@ -3337,14 +3338,7 @@ def test_anthropic_us_data_residency_uplift_on_claude_4_6_and_later_models(_loca
|
|||
|
||||
|
||||
def test_gemini_cache_tokens_details_no_negative_values():
|
||||
"""
|
||||
Test for Issue #18750: Negative text_tokens with Gemini caching
|
||||
|
||||
When using Gemini with explicit caching, the response includes cacheTokensDetails
|
||||
which breaks down cached tokens by modality. This test ensures that:
|
||||
1. text_tokens is never negative
|
||||
2. We correctly subtract cached tokens per modality (not total)
|
||||
"""
|
||||
"""Ensure issue #18750 cache details remain non-negative after cost parsing."""
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
VertexGeminiConfig,
|
||||
)
|
||||
|
|
@ -3372,27 +3366,17 @@ def test_gemini_cache_tokens_details_no_negative_values():
|
|||
|
||||
usage = VertexGeminiConfig.calculate_usage(completion_response)
|
||||
|
||||
# Text tokens should be non-cached text only: 9402 - 9393 = 9
|
||||
assert usage.prompt_tokens_details.text_tokens == 9, (
|
||||
f"Expected text_tokens=9, got {usage.prompt_tokens_details.text_tokens}"
|
||||
)
|
||||
prompt_details: Final = parse_prompt_tokens_details(usage)
|
||||
|
||||
# Image tokens should be non-cached image only: 258 - 258 = 0
|
||||
assert usage.prompt_tokens_details.image_tokens == 0, (
|
||||
f"Expected image_tokens=0, got {usage.prompt_tokens_details.image_tokens}"
|
||||
assert usage.prompt_tokens_details.text_tokens == 9402
|
||||
assert usage.prompt_tokens_details.image_tokens == 258
|
||||
assert usage.prompt_tokens_details.cached_tokens == 9651
|
||||
assert usage.prompt_tokens_details.cached_tokens_details == CachedTokensDetails(
|
||||
text_tokens=9393, image_tokens=258
|
||||
)
|
||||
|
||||
# Total cached should match
|
||||
assert usage.prompt_tokens_details.cached_tokens == 9651, (
|
||||
f"Expected cached_tokens=9651, got {usage.prompt_tokens_details.cached_tokens}"
|
||||
)
|
||||
|
||||
# MOST IMPORTANT: text_tokens should NEVER be negative
|
||||
assert usage.prompt_tokens_details.text_tokens >= 0, (
|
||||
f"BUG: text_tokens is negative ({usage.prompt_tokens_details.text_tokens})! This was the issue in #18750"
|
||||
)
|
||||
|
||||
print("✅ Issue #18750 fix verified: text_tokens is correctly calculated and non-negative")
|
||||
assert prompt_details["text_tokens"] == 9
|
||||
assert prompt_details["image_tokens"] == 0
|
||||
assert prompt_details["text_tokens"] >= 0
|
||||
|
||||
|
||||
def test_gemini_without_cache_tokens_details():
|
||||
|
|
@ -5990,6 +5974,45 @@ def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_t
|
|||
assert prompt_cost == pytest.approx(298_926 * 3e-6 + 64 * 5e-6 + 10 * 3e-6 + 1_000 * 3e-7)
|
||||
|
||||
|
||||
def test_batch_cost_calculator_prices_cached_audio_at_its_batch_rate() -> None:
|
||||
model_info: Final[ModelInfo] = {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"input_cost_per_token_batches": 1e-6,
|
||||
"input_cost_per_audio_token": 6e-6,
|
||||
"input_cost_per_audio_token_batches": 3e-6,
|
||||
"cache_read_input_token_cost": 8e-7,
|
||||
"cache_read_input_token_cost_batches": 4e-7,
|
||||
"cache_read_input_audio_token_cost": 5e-7,
|
||||
"cache_read_input_audio_token_cost_batches": 2e-7,
|
||||
}
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=40,
|
||||
completion_tokens=0,
|
||||
total_tokens=40,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=20,
|
||||
audio_tokens=20,
|
||||
cached_tokens=10,
|
||||
cached_tokens_details=CachedTokensDetails(text_tokens=5, audio_tokens=5),
|
||||
),
|
||||
)
|
||||
|
||||
prompt_cost, _ = batch_cost_calculator(
|
||||
usage=usage,
|
||||
model="gemini-audio-batch-test-model",
|
||||
custom_llm_provider="gemini",
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
expected: Final = (
|
||||
15 * model_info["input_cost_per_token_batches"]
|
||||
+ 15 * model_info["input_cost_per_audio_token_batches"]
|
||||
+ 5 * model_info["cache_read_input_token_cost_batches"]
|
||||
+ 5 * model_info["cache_read_input_audio_token_cost_batches"]
|
||||
)
|
||||
assert prompt_cost == pytest.approx(expected)
|
||||
|
||||
|
||||
QWEN3_NEXT_REGIONS: Final = ("ap-northeast-1", "ap-south-1", "ap-southeast-2", "eu-west-1", "eu-west-2", "sa-east-1")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -791,3 +791,26 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa
|
|||
sibling = rows["volcengine/doubao-seed-2-1-pro-260628"]
|
||||
assert isinstance(sibling, dict)
|
||||
assert all(entry[key] == sibling[key] for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost"))
|
||||
|
||||
|
||||
def _bedrock_gpt_oss_rows(prices: Mapping[str, JsonValue]) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
key
|
||||
for key, row in prices.items()
|
||||
if "gpt-oss" in key and isinstance(row, dict) and str(row.get("litellm_provider", "")).startswith("bedrock")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", [PRICES_PATH, BACKUP_PRICES_PATH], ids=["main", "backup"])
|
||||
def test_bedrock_gpt_oss_rows_declare_vision_support_like_their_safeguard_siblings(path: Path) -> None:
|
||||
prices: Final = json.loads(path.read_text())
|
||||
rows: Final = _bedrock_gpt_oss_rows(prices)
|
||||
assert rows, "no Bedrock gpt-oss rows found"
|
||||
reference: Final = prices["openai.gpt-oss-safeguard-120b"]["supports_vision"]
|
||||
undeclared: Final = tuple(key for key in rows if prices[key].get("supports_vision") is not reference)
|
||||
assert undeclared == (), f"modality routing only skips rows that declare supports_vision: {undeclared}"
|
||||
|
||||
|
||||
def test_bedrock_gpt_oss_vision_is_explicitly_disabled_for_the_router_gate() -> None:
|
||||
assert litellm.utils.is_vision_explicitly_disabled("bedrock/openai.gpt-oss-120b-1:0") is True
|
||||
assert litellm.utils.is_vision_explicitly_disabled("bedrock_mantle/openai.gpt-oss-20b") is True
|
||||
|
|
|
|||
|
|
@ -29871,3 +29871,105 @@ async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_s
|
|||
wrapped.adopt_fallback_headers(fallback)
|
||||
|
||||
assert wrapped.partial_response_on_disconnect().response.usage.output_tokens == primary_tokens + fallback_tokens
|
||||
|
||||
|
||||
_IMAGE_PART = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}
|
||||
_IMAGE_CHAT = [{"role": "user", "content": [{"type": "text", "text": "What is in this image?"}, _IMAGE_PART]}]
|
||||
_IMAGE_RESPONSES_INPUT = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "What is in this image?"},
|
||||
{"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo="},
|
||||
],
|
||||
}
|
||||
]
|
||||
_VISION_LEG: Final[dict[str, object]] = {"litellm_params": {"model": "openai/gpt-5.6-terra"}, "model_info": {"id": "vision"}}
|
||||
_DECLARED_TEXT_LEG: Final[dict[str, object]] = {
|
||||
"litellm_params": {"model": "bedrock/openai.gpt-oss-120b-1:0"},
|
||||
"model_info": {"id": "declared-text", "supports_vision": False},
|
||||
}
|
||||
|
||||
|
||||
def _pre_call_router(*deployments: dict[str, object], routing_strategy: str = "latency-based-routing") -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[{"model_name": "mixed", **deployment} for deployment in deployments],
|
||||
routing_strategy=routing_strategy,
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
|
||||
|
||||
def _pre_call_ids(router: litellm.Router, deployments: tuple[dict[str, object], ...], **request: object) -> list[str]:
|
||||
return [d["model_info"]["id"] for d in router._pre_call_checks(model="mixed", healthy_deployments=list(deployments), **request)]
|
||||
|
||||
|
||||
def test_pre_call_checks_drop_declared_text_only_legs_for_image_requests():
|
||||
router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=_IMAGE_CHAT) == ["vision"]
|
||||
|
||||
|
||||
def test_pre_call_checks_keep_declared_text_only_legs_for_text_requests():
|
||||
router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=[{"role": "user", "content": "Say hi"}]) == [
|
||||
"vision",
|
||||
"declared-text",
|
||||
]
|
||||
|
||||
|
||||
def test_pre_call_checks_drop_declared_text_only_legs_for_responses_input_image():
|
||||
router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
assert _pre_call_ids(
|
||||
router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=None, request_kwargs={"input": _IMAGE_RESPONSES_INPUT}
|
||||
) == ["vision"]
|
||||
|
||||
|
||||
def test_pre_call_checks_send_image_to_the_group_when_every_leg_is_declared_text_only():
|
||||
other_text_leg = {**_DECLARED_TEXT_LEG, "model_info": {"id": "declared-text-2", "supports_vision": False}}
|
||||
router = _pre_call_router(_DECLARED_TEXT_LEG, other_text_leg)
|
||||
assert _pre_call_ids(router, (_DECLARED_TEXT_LEG, other_text_leg), messages=_IMAGE_CHAT) == [
|
||||
"declared-text",
|
||||
"declared-text-2",
|
||||
]
|
||||
|
||||
|
||||
def test_pre_call_checks_keep_undeclared_custom_legs_for_image_requests():
|
||||
custom_leg = {
|
||||
"litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct", "api_base": "http://localhost:8000"},
|
||||
"model_info": {"id": "custom"},
|
||||
}
|
||||
router = _pre_call_router(_VISION_LEG, custom_leg)
|
||||
assert _pre_call_ids(router, (_VISION_LEG, custom_leg), messages=_IMAGE_CHAT) == ["vision", "custom"]
|
||||
|
||||
|
||||
def test_pre_call_checks_resolve_vision_through_the_cost_map_and_base_model():
|
||||
catalog_text_leg = {"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {"id": "catalog-text"}}
|
||||
azure_text_leg = {
|
||||
"litellm_params": {"model": "azure/my-audio-deployment", "api_base": "https://x.openai.azure.com"},
|
||||
"model_info": {"id": "azure-text", "base_model": "azure/gpt-audio"},
|
||||
}
|
||||
router = _pre_call_router(_VISION_LEG, catalog_text_leg, azure_text_leg)
|
||||
assert _pre_call_ids(router, (_VISION_LEG, catalog_text_leg, azure_text_leg), messages=_IMAGE_CHAT) == ["vision"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment", "expected"),
|
||||
[
|
||||
(_DECLARED_TEXT_LEG, True),
|
||||
({"litellm_params": {"model": "bedrock/openai.gpt-oss-120b-1:0"}, "model_info": {"supports_vision": True}}, False),
|
||||
({"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {}}, True),
|
||||
({"litellm_params": {"model": "azure/my-deploy"}, "model_info": {"base_model": "azure/gpt-audio"}}, True),
|
||||
({"litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct"}, "model_info": {}}, False),
|
||||
({"litellm_params": {"model": "no-such-model-anywhere"}, "model_info": {}}, False),
|
||||
({"litellm_params": {}, "model_info": {}}, False),
|
||||
],
|
||||
ids=["override-false", "override-true", "cost-map-false", "base-model-false", "undeclared", "unresolvable", "no-params"],
|
||||
)
|
||||
def test_deployment_declares_no_vision_reads_the_override_then_the_cost_map(deployment: dict[str, object], expected: bool):
|
||||
assert litellm.Router._deployment_declares_no_vision(deployment, "mixed") is expected
|
||||
|
||||
|
||||
def test_request_carries_image_sees_chat_parts_and_responses_input_items():
|
||||
assert litellm.Router._request_carries_image(_IMAGE_CHAT, {}) is True
|
||||
assert litellm.Router._request_carries_image(None, {"input": _IMAGE_RESPONSES_INPUT}) is True
|
||||
assert litellm.Router._request_carries_image([{"role": "user", "content": "Say hi"}], {}) is False
|
||||
assert litellm.Router._request_carries_image(None, {"input": "Say hi"}) is False
|
||||
|
|
|
|||
|
|
@ -939,6 +939,18 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"cache_creation_input_audio_token_cost": {"type": "number"},
|
||||
"cache_creation_input_token_cost": {"type": "number"},
|
||||
"cache_creation_input_token_cost_above_1hr": {"type": "number"},
|
||||
"cache_creation_input_token_cost_above_12hr": {"type": "number"},
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens": {
|
||||
"type": "number"
|
||||
},
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": {
|
||||
"type": "number"
|
||||
},
|
||||
"cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": {
|
||||
"type": "number"
|
||||
},
|
||||
"cache_creation_input_token_cost_above_12hr_flex": {"type": "number"},
|
||||
"cache_creation_input_token_cost_above_12hr_priority": {"type": "number"},
|
||||
"cache_creation_input_token_cost_above_32k_tokens": {"type": "number"},
|
||||
"cache_creation_input_token_cost_above_100k_tokens": {"type": "number"},
|
||||
"cache_creation_input_token_cost_above_100k_tokens_batches": {"type": "number"},
|
||||
|
|
@ -982,6 +994,9 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": {"type": "number"},
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"},
|
||||
"cache_read_input_audio_token_cost": {"type": "number"},
|
||||
"cache_read_input_audio_token_cost_batches": {"type": "number"},
|
||||
"cache_read_input_audio_token_cost_flex": {"type": "number"},
|
||||
"cache_read_input_audio_token_cost_priority": {"type": "number"},
|
||||
"cache_read_input_image_token_cost": {"type": "number"},
|
||||
"audio_transcription_config": {"type": "string"},
|
||||
"deprecation_date": {"type": "string"},
|
||||
|
|
@ -989,6 +1004,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"},
|
||||
"google_maps_grounding_cost_per_query": {"type": "number"},
|
||||
"input_cost_per_audio_token": {"type": "number"},
|
||||
"input_cost_per_audio_token_flex": {"type": "number"},
|
||||
"input_cost_per_image_token": {"type": "number"},
|
||||
"input_cost_per_character": {"type": "number"},
|
||||
"input_cost_per_character_above_128k_tokens": {"type": "number"},
|
||||
|
|
|
|||
|
|
@ -61,6 +61,30 @@ def test_get_model_info_custom_llm_with_same_name_vllm(monkeypatch):
|
|||
assert model_info["input_cost_per_token"] == 0.0
|
||||
|
||||
|
||||
def test_get_model_info_passes_through_gemini_tiered_audio_rates() -> None:
|
||||
model: Final = "gemini-tiered-audio-rates-test"
|
||||
prices: Final = {
|
||||
"input_cost_per_audio_token_flex": 1e-6,
|
||||
"input_cost_per_audio_token_priority": 2e-6,
|
||||
"cache_read_input_audio_token_cost_flex": 3e-7,
|
||||
"cache_read_input_audio_token_cost_priority": 4e-7,
|
||||
"cache_read_input_audio_token_cost_batches": 5e-7,
|
||||
}
|
||||
litellm.register_model(
|
||||
{
|
||||
model: {
|
||||
"litellm_provider": "gemini",
|
||||
"mode": "chat",
|
||||
**prices,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="gemini")
|
||||
|
||||
assert {key: model_info[key] for key in prices} == prices
|
||||
|
||||
|
||||
def test_get_model_info_ollama_chat():
|
||||
from litellm.llms.ollama.completion.transformation import OllamaConfig
|
||||
|
||||
|
|
|
|||
72
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
72
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -28439,6 +28439,15 @@ export interface components {
|
|||
*/
|
||||
uncached_input_tokens: number;
|
||||
};
|
||||
/** CacheWriteTokensDetails */
|
||||
CacheWriteTokensDetails: {
|
||||
/** Ttl 12H Tokens */
|
||||
ttl_12h_tokens: number;
|
||||
/** Ttl 30M Tokens */
|
||||
ttl_30m_tokens: number;
|
||||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** CachedTokensDetails */
|
||||
CachedTokensDetails: {
|
||||
/** Audio Tokens */
|
||||
|
|
@ -33589,6 +33598,9 @@ export interface components {
|
|||
InputTokensDetails: {
|
||||
/** Audio Tokens */
|
||||
audio_tokens?: number | null;
|
||||
/** Cache Write Tokens */
|
||||
cache_write_tokens?: number | null;
|
||||
cache_write_tokens_details?: components["schemas"]["CacheWriteTokensDetails"] | null;
|
||||
/**
|
||||
* Cached Tokens
|
||||
* @default 0
|
||||
|
|
@ -35207,6 +35219,18 @@ export interface components {
|
|||
cache_creation_input_token_cost_above_100k_tokens?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 100K Tokens Batches */
|
||||
cache_creation_input_token_cost_above_100k_tokens_batches?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr */
|
||||
cache_creation_input_token_cost_above_12hr?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Flex */
|
||||
cache_creation_input_token_cost_above_12hr_flex?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Priority */
|
||||
cache_creation_input_token_cost_above_12hr_priority?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 1Hr */
|
||||
cache_creation_input_token_cost_above_1hr?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */
|
||||
|
|
@ -35235,6 +35259,12 @@ export interface components {
|
|||
cache_creation_input_token_cost_ultrafast?: number | null;
|
||||
/** Cache Read Input Audio Token Cost */
|
||||
cache_read_input_audio_token_cost?: number | null;
|
||||
/** Cache Read Input Audio Token Cost Batches */
|
||||
cache_read_input_audio_token_cost_batches?: number | null;
|
||||
/** Cache Read Input Audio Token Cost Flex */
|
||||
cache_read_input_audio_token_cost_flex?: number | null;
|
||||
/** Cache Read Input Audio Token Cost Priority */
|
||||
cache_read_input_audio_token_cost_priority?: number | null;
|
||||
/** Cache Read Input Image Token Cost */
|
||||
cache_read_input_image_token_cost?: number | null;
|
||||
/** Cache Read Input Token Cost */
|
||||
|
|
@ -35314,6 +35344,10 @@ export interface components {
|
|||
input_cost_per_audio_token?: number | null;
|
||||
/** Input Cost Per Audio Token Batches */
|
||||
input_cost_per_audio_token_batches?: number | null;
|
||||
/** Input Cost Per Audio Token Flex */
|
||||
input_cost_per_audio_token_flex?: number | null;
|
||||
/** Input Cost Per Audio Token Priority */
|
||||
input_cost_per_audio_token_priority?: number | null;
|
||||
/** Input Cost Per Character */
|
||||
input_cost_per_character?: number | null;
|
||||
/** Input Cost Per Character Above 128K Tokens */
|
||||
|
|
@ -41506,9 +41540,9 @@ export interface components {
|
|||
mode?: "implicit" | "explicit";
|
||||
/**
|
||||
* Ttl
|
||||
* @constant
|
||||
* @enum {string}
|
||||
*/
|
||||
ttl?: "30m";
|
||||
ttl?: "30m" | "12h";
|
||||
};
|
||||
/** PromptCachingRequest */
|
||||
PromptCachingRequest: {
|
||||
|
|
@ -50199,6 +50233,18 @@ export interface components {
|
|||
blocked?: boolean | null;
|
||||
/** Cache Creation Input Token Cost */
|
||||
cache_creation_input_token_cost?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr */
|
||||
cache_creation_input_token_cost_above_12hr?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Flex */
|
||||
cache_creation_input_token_cost_above_12hr_flex?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Priority */
|
||||
cache_creation_input_token_cost_above_12hr_priority?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 1Hr */
|
||||
cache_creation_input_token_cost_above_1hr?: number | null;
|
||||
/** Cache Read Input Token Cost */
|
||||
|
|
@ -50492,6 +50538,18 @@ export interface components {
|
|||
cache_creation_input_token_cost_above_100k_tokens?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 100K Tokens Batches */
|
||||
cache_creation_input_token_cost_above_100k_tokens_batches?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr */
|
||||
cache_creation_input_token_cost_above_12hr?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens */
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Flex */
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Above 272K Tokens Priority */
|
||||
cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Flex */
|
||||
cache_creation_input_token_cost_above_12hr_flex?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 12Hr Priority */
|
||||
cache_creation_input_token_cost_above_12hr_priority?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 1Hr */
|
||||
cache_creation_input_token_cost_above_1hr?: number | null;
|
||||
/** Cache Creation Input Token Cost Above 1Hr Above 100K Tokens */
|
||||
|
|
@ -50520,6 +50578,12 @@ export interface components {
|
|||
cache_creation_input_token_cost_ultrafast?: number | null;
|
||||
/** Cache Read Input Audio Token Cost */
|
||||
cache_read_input_audio_token_cost?: number | null;
|
||||
/** Cache Read Input Audio Token Cost Batches */
|
||||
cache_read_input_audio_token_cost_batches?: number | null;
|
||||
/** Cache Read Input Audio Token Cost Flex */
|
||||
cache_read_input_audio_token_cost_flex?: number | null;
|
||||
/** Cache Read Input Audio Token Cost Priority */
|
||||
cache_read_input_audio_token_cost_priority?: number | null;
|
||||
/** Cache Read Input Image Token Cost */
|
||||
cache_read_input_image_token_cost?: number | null;
|
||||
/** Cache Read Input Token Cost */
|
||||
|
|
@ -50599,6 +50663,10 @@ export interface components {
|
|||
input_cost_per_audio_token?: number | null;
|
||||
/** Input Cost Per Audio Token Batches */
|
||||
input_cost_per_audio_token_batches?: number | null;
|
||||
/** Input Cost Per Audio Token Flex */
|
||||
input_cost_per_audio_token_flex?: number | null;
|
||||
/** Input Cost Per Audio Token Priority */
|
||||
input_cost_per_audio_token_priority?: number | null;
|
||||
/** Input Cost Per Character */
|
||||
input_cost_per_character?: number | null;
|
||||
/** Input Cost Per Character Above 128K Tokens */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue