mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
Merge 11937bf88d into 86ef1fb08b
This commit is contained in:
LH-kevin
GitHub
commit
ecd213640b
8 changed files with 1211 additions and 135 deletions
|
|
@ -1,6 +1,7 @@
|
|||
# What is this?
|
||||
## Helper utilities for cost_per_token()
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
|
|
@ -48,6 +49,15 @@ _SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple(
|
|||
sorted((f"_{st.value}" for st in ServiceTier), key=len, reverse=True)
|
||||
)
|
||||
|
||||
_ABOVE_TOKEN_THRESHOLD_COST_KEY: Final[re.Pattern[str]] = re.compile(
|
||||
r"^(?P<base_key>"
|
||||
r"input_cost_per_token|"
|
||||
r"output_cost_per_token|"
|
||||
r"cache_creation_input_token_cost(?:_above_1hr)?|"
|
||||
r"cache_read_input_token_cost"
|
||||
r")_above_\d+k?_tokens(?:_(?P<tier>priority|flex|ultrafast))?$"
|
||||
)
|
||||
|
||||
_SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
ServiceTier.FLEX.value: ServiceTier.FLEX.value,
|
||||
|
|
@ -211,7 +221,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str:
|
|||
|
||||
|
||||
def _parse_above_token_threshold(key: str) -> float:
|
||||
threshold_str: Final = key.split("_above_")[1].split("_tokens")[0]
|
||||
threshold_str: Final = key.rsplit("_above_", 1)[1].split("_tokens")[0]
|
||||
return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1)
|
||||
|
||||
|
||||
|
|
@ -276,8 +286,9 @@ def _get_token_base_cost(
|
|||
"""
|
||||
Return prompt cost, completion cost, and cache costs for a given model and usage.
|
||||
|
||||
If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set,
|
||||
then we use the corresponding threshold cost for all token types.
|
||||
For each token cost type, use its highest configured
|
||||
`*_above_[x]k_tokens` or `*_above_[x]_tokens` rate when prompt tokens
|
||||
exceed that threshold.
|
||||
|
||||
`threshold_is_inclusive` switches that comparison to >=, for providers such as xAI
|
||||
that bill the higher tier once the prompt reaches the threshold.
|
||||
|
|
@ -312,127 +323,70 @@ def _get_token_base_cost(
|
|||
cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key))
|
||||
|
||||
## CHECK IF ABOVE THRESHOLD
|
||||
# Optimization: collect threshold keys first to avoid sorting all model_info keys.
|
||||
# Most models don't have threshold pricing, so we can return early.
|
||||
# Exclude service_tier-specific variants (e.g. input_cost_per_token_above_200k_tokens_priority)
|
||||
# so that the threshold detection loop only processes standard keys. The
|
||||
# service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key.
|
||||
threshold_keys: Final = [
|
||||
k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES)
|
||||
]
|
||||
if not threshold_keys:
|
||||
return (
|
||||
prompt_base_cost,
|
||||
completion_base_cost,
|
||||
cache_creation_cost,
|
||||
cache_creation_cost_above_1hr,
|
||||
cache_read_cost,
|
||||
selected_threshold_keys: dict[str, tuple[float, str, str | None]] = {} # mutable-ok: threshold accumulator
|
||||
|
||||
for key in model_info:
|
||||
if "_above_" not in key or not key.endswith(("_tokens", "_priority", "_flex", "_ultrafast")):
|
||||
continue
|
||||
|
||||
match = _ABOVE_TOKEN_THRESHOLD_COST_KEY.fullmatch(key)
|
||||
if match is None or model_info.get(key) is None:
|
||||
continue
|
||||
|
||||
threshold_key_tier = match.group("tier")
|
||||
active_pricing_tier = (
|
||||
None if service_tier is None else _SERVICE_TIER_TO_COST_KEY_SUFFIX.get(service_tier.lower())
|
||||
)
|
||||
if threshold_key_tier is not None and threshold_key_tier != active_pricing_tier:
|
||||
continue
|
||||
|
||||
try:
|
||||
threshold = _parse_above_token_threshold(key)
|
||||
except (IndexError, ValueError):
|
||||
continue
|
||||
|
||||
if usage.prompt_tokens < threshold or (usage.prompt_tokens == threshold and not threshold_is_inclusive):
|
||||
continue
|
||||
|
||||
base_key = match.group("base_key")
|
||||
selected = selected_threshold_keys.get(base_key)
|
||||
selected_specificity = 0 if selected is None or selected[2] is None else 1
|
||||
candidate_specificity = 0 if threshold_key_tier is None else 1
|
||||
|
||||
if selected is None or (threshold, candidate_specificity) > (selected[0], selected_specificity):
|
||||
selected_threshold_keys[base_key] = (
|
||||
threshold,
|
||||
key,
|
||||
threshold_key_tier,
|
||||
)
|
||||
|
||||
costs_by_base_key = { # mutable-ok: selected-rate updates
|
||||
"input_cost_per_token": prompt_base_cost,
|
||||
"output_cost_per_token": completion_base_cost,
|
||||
"cache_creation_input_token_cost": cache_creation_cost,
|
||||
"cache_creation_input_token_cost_above_1hr": (cache_creation_cost_above_1hr),
|
||||
"cache_read_input_token_cost": cache_read_cost,
|
||||
}
|
||||
|
||||
for base_key, (_, threshold_key, threshold_key_tier) in selected_threshold_keys.items():
|
||||
cost_key = (
|
||||
threshold_key if threshold_key_tier is not None else _get_service_tier_cost_key(threshold_key, service_tier)
|
||||
)
|
||||
costs_by_base_key[base_key] = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info,
|
||||
cost_key,
|
||||
costs_by_base_key[base_key],
|
||||
),
|
||||
)
|
||||
|
||||
# Only sort the threshold keys (typically 1-2 keys instead of 66+)
|
||||
threshold: float | None = None
|
||||
for key in sorted(threshold_keys, key=_parse_above_token_threshold, reverse=True):
|
||||
value = model_info.get(key)
|
||||
if value is not None:
|
||||
try:
|
||||
# Handle both formats: _above_128k_tokens and _above_128_tokens
|
||||
threshold_str = key.split("_above_")[1].split("_tokens")[0]
|
||||
threshold = _parse_above_token_threshold(key)
|
||||
if usage.prompt_tokens > threshold or (threshold_is_inclusive and usage.prompt_tokens == threshold):
|
||||
# Prefer a service_tier-specific above-threshold key when available,
|
||||
# e.g. input_cost_per_token_priority_above_200k_tokens for Gemini
|
||||
# ON_DEMAND_PRIORITY. Falls back to the standard key automatically
|
||||
# via _get_cost_per_unit's service_tier fallback logic.
|
||||
tiered_input_key = (
|
||||
_get_service_tier_cost_key(
|
||||
f"input_cost_per_token_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else key
|
||||
)
|
||||
prompt_base_cost = cast(
|
||||
float,
|
||||
_get_cost_per_unit(model_info, tiered_input_key, prompt_base_cost),
|
||||
)
|
||||
tiered_output_key = (
|
||||
_get_service_tier_cost_key(
|
||||
f"output_cost_per_token_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else f"output_cost_per_token_above_{threshold_str}_tokens"
|
||||
)
|
||||
completion_base_cost = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info,
|
||||
tiered_output_key,
|
||||
completion_base_cost,
|
||||
),
|
||||
)
|
||||
|
||||
# Apply tiered pricing to cache costs
|
||||
cache_creation_tiered_key = (
|
||||
_get_service_tier_cost_key(
|
||||
f"cache_creation_input_token_cost_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
|
||||
)
|
||||
cache_creation_1hr_tiered_key = (
|
||||
_get_service_tier_cost_key(
|
||||
f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
|
||||
)
|
||||
cache_read_tiered_key = (
|
||||
_get_service_tier_cost_key(
|
||||
f"cache_read_input_token_cost_above_{threshold_str}_tokens",
|
||||
service_tier,
|
||||
)
|
||||
if service_tier
|
||||
else f"cache_read_input_token_cost_above_{threshold_str}_tokens"
|
||||
)
|
||||
|
||||
cache_creation_cost = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info,
|
||||
cache_creation_tiered_key,
|
||||
cache_creation_cost,
|
||||
),
|
||||
)
|
||||
|
||||
cache_creation_cost_above_1hr = cast(
|
||||
float,
|
||||
_get_cost_per_unit(
|
||||
model_info,
|
||||
cache_creation_1hr_tiered_key,
|
||||
cache_creation_cost_above_1hr,
|
||||
),
|
||||
)
|
||||
|
||||
cache_read_cost = cast(
|
||||
float,
|
||||
_get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost),
|
||||
)
|
||||
|
||||
break
|
||||
except (IndexError, ValueError):
|
||||
continue
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return (
|
||||
prompt_base_cost,
|
||||
completion_base_cost,
|
||||
cache_creation_cost,
|
||||
cache_creation_cost_above_1hr,
|
||||
cache_read_cost,
|
||||
costs_by_base_key["input_cost_per_token"],
|
||||
costs_by_base_key["output_cost_per_token"],
|
||||
costs_by_base_key["cache_creation_input_token_cost"],
|
||||
costs_by_base_key["cache_creation_input_token_cost_above_1hr"],
|
||||
costs_by_base_key["cache_read_input_token_cost"],
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ together because they have to agree: a deployment the rollup declines to charge
|
|||
router prices at zero serves its traffic for free.
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, time, timezone
|
||||
|
|
@ -43,6 +44,26 @@ SEARCH_CONTEXT_SIZES: Final = ("search_context_size_low", "search_context_size_m
|
|||
# and zeroing one of those would destroy the deployment's configuration rather than stop a
|
||||
# charge.
|
||||
CUSTOM_PRICING_FIELDS: Final = frozenset(f for f in CustomPricingLiteLLMParams.model_fields if "cost" in f)
|
||||
# Custom threshold rates the router copies from litellm_params into the cost map (e.g.
|
||||
# input_cost_per_token_above_32k_tokens) are not enumerated in CustomPricingLiteLLMParams,
|
||||
# so they are matched here and zeroed like every other declared rate. Only the cost
|
||||
# calculator's base keys qualify, so a param that merely ends in _above_<N>k_tokens (e.g.
|
||||
# api_key_above_32k_tokens) is not treated as a charge.
|
||||
_THRESHOLD_RATE_KEY: Final[re.Pattern[str]] = re.compile(
|
||||
r"^(?:"
|
||||
r"input_cost_per_token|"
|
||||
r"output_cost_per_token|"
|
||||
r"cache_creation_input_token_cost(?:_above_1hr)?|"
|
||||
r"cache_read_input_token_cost"
|
||||
r")_above_\d+k?_tokens(?:_(?:priority|flex|ultrafast))?$"
|
||||
)
|
||||
|
||||
|
||||
def is_threshold_rate_key(field: str) -> bool:
|
||||
"""Whether ``field`` is a custom above-threshold rate key the cost calculator reads."""
|
||||
return _THRESHOLD_RATE_KEY.fullmatch(field) is not None
|
||||
|
||||
|
||||
PTU_ZEROED_PRICING: Final[Mapping[str, float | tuple[()] | Mapping[str, float]]] = MappingProxyType(
|
||||
{
|
||||
**dict.fromkeys(PTU_ZEROED_PRICING_FIELDS, 0.0),
|
||||
|
|
@ -224,6 +245,7 @@ def zeroed_ptu_pricing(
|
|||
return None
|
||||
if not is_ptu_cost_attribution_enabled():
|
||||
return None
|
||||
declared_threshold_rates: Final = frozenset(key for key in declared if is_threshold_rate_key(key))
|
||||
return MappingProxyType(
|
||||
{
|
||||
**PTU_ZEROED_PRICING,
|
||||
|
|
@ -233,5 +255,6 @@ def zeroed_ptu_pricing(
|
|||
.difference(PTU_EMPTIED_PRICING_FIELDS),
|
||||
0.0,
|
||||
),
|
||||
**dict.fromkeys(declared_threshold_rates, 0.0),
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from litellm.litellm_core_utils.ptu_pricing import (
|
|||
PTU_ZEROED_PRICING_FIELDS,
|
||||
PTU_ZEROED_TABLE_FIELDS,
|
||||
SEARCH_CONTEXT_SIZES,
|
||||
is_threshold_rate_key,
|
||||
ptu_config_error,
|
||||
)
|
||||
from litellm.proxy._types import (
|
||||
|
|
@ -420,6 +421,9 @@ def _raise_if_ptu_deployment_is_priced(*, model_info: Mapping[str, object], supp
|
|||
sorted(
|
||||
tuple(field for field in _CUSTOM_PRICING_FIELDS if _is_nonzero_price(supplied.get(field)))
|
||||
+ tuple(field for field in _PTU_EMPTIED_PRICING_FIELDS if supplied.get(field))
|
||||
+ tuple(
|
||||
field for field in supplied if is_threshold_rate_key(field) and _is_nonzero_price(supplied.get(field))
|
||||
)
|
||||
)
|
||||
)
|
||||
if not priced:
|
||||
|
|
@ -459,9 +463,12 @@ def _ptu_zeroed_pricing(
|
|||
if model_info.get("ptu_count") is None or model_info.get("cost_per_ptu_per_hour") is None:
|
||||
return _NO_PRICING_OVERRIDE
|
||||
_raise_if_ptu_deployment_is_priced(model_info=model_info, supplied=supplied)
|
||||
candidates: Final = _CUSTOM_PRICING_FIELDS | frozenset(
|
||||
field for field in (*model_info.keys(), *litellm_params.keys()) if is_threshold_rate_key(field)
|
||||
)
|
||||
stored: Final = frozenset(
|
||||
field
|
||||
for field in _CUSTOM_PRICING_FIELDS
|
||||
for field in candidates
|
||||
if _is_nonzero_price(model_info.get(field)) or _is_nonzero_price(litellm_params.get(field))
|
||||
)
|
||||
return MappingProxyType(
|
||||
|
|
@ -501,7 +508,11 @@ def _ptu_pricing_delta(
|
|||
return _NO_PRICING_OVERRIDE, frozenset()
|
||||
return _NO_PRICING_OVERRIDE, frozenset(
|
||||
field
|
||||
for field in _CUSTOM_PRICING_FIELDS.union(_PTU_ZEROED_PRICING_FIELDS, _PTU_EMPTIED_PRICING_FIELDS)
|
||||
for field in _CUSTOM_PRICING_FIELDS.union(
|
||||
_PTU_ZEROED_PRICING_FIELDS,
|
||||
_PTU_EMPTIED_PRICING_FIELDS,
|
||||
frozenset(field for field in (*model_info.keys(), *litellm_params.keys()) if is_threshold_rate_key(field)),
|
||||
)
|
||||
if _is_zero_price(model_info.get(field)) or _is_zero_price(litellm_params.get(field))
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -304,6 +304,36 @@ def _cost_value_as_float(value: str | float | None) -> float | None:
|
|||
return None
|
||||
|
||||
|
||||
_CUSTOM_PRICING_THRESHOLD_COST_KEY: Final[re.Pattern[str]] = re.compile(
|
||||
r"^(?:"
|
||||
r"input_cost_per_token|"
|
||||
r"output_cost_per_token|"
|
||||
r"cache_creation_input_token_cost(?:_above_1hr)?|"
|
||||
r"cache_read_input_token_cost"
|
||||
r")_above_\d+k?_tokens(?:_(?:priority|flex|ultrafast))?$"
|
||||
)
|
||||
|
||||
|
||||
def _is_custom_pricing_field(field: str) -> bool:
|
||||
return (
|
||||
field in CustomPricingLiteLLMParams.model_fields or _CUSTOM_PRICING_THRESHOLD_COST_KEY.search(field) is not None
|
||||
)
|
||||
|
||||
|
||||
def _copy_custom_pricing_fields(
|
||||
model_info: dict[str, Any], # mutable-ok: updates model-cost entry in place
|
||||
litellm_params: LiteLLM_Params,
|
||||
) -> None:
|
||||
"""Copy declared and arbitrary threshold pricing into a model-cost entry."""
|
||||
model_info.update(
|
||||
{
|
||||
field: value
|
||||
for field, value in litellm_params.model_dump(exclude_none=True).items()
|
||||
if _is_custom_pricing_field(field)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool:
|
||||
"""Single owner of the environment-gating rule: a deployment whose model_info names
|
||||
`supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list.
|
||||
|
|
@ -8170,9 +8200,10 @@ class Router:
|
|||
litellm_params=litellm_params,
|
||||
model_info=_model_info,
|
||||
)
|
||||
for field in CustomPricingLiteLLMParams.model_fields:
|
||||
if deployment.litellm_params.get(field) is not None:
|
||||
_model_info[field] = deployment.litellm_params[field]
|
||||
_copy_custom_pricing_fields(
|
||||
model_info=_model_info,
|
||||
litellm_params=deployment.litellm_params,
|
||||
)
|
||||
|
||||
if _model_info.get("input_cost_per_token") is not None:
|
||||
Router._inherit_builtin_cache_pricing(
|
||||
|
|
@ -8910,10 +8941,10 @@ class Router:
|
|||
self._add_deployment(deployment=deployment)
|
||||
|
||||
_model_info_dict: Final[dict] = deployment.model_info.model_dump(exclude_none=True)
|
||||
for field in CustomPricingLiteLLMParams.model_fields:
|
||||
field_value = deployment.litellm_params.get(field)
|
||||
if field_value is not None:
|
||||
_model_info_dict[field] = field_value
|
||||
_copy_custom_pricing_fields(
|
||||
model_info=_model_info_dict,
|
||||
litellm_params=deployment.litellm_params,
|
||||
)
|
||||
|
||||
if _model_info_dict.get("input_cost_per_token") is not None:
|
||||
Router._inherit_builtin_cache_pricing(
|
||||
|
|
@ -9165,10 +9196,10 @@ class Router:
|
|||
deployment alone, which is what lets a refresh rebuild the same entries.
|
||||
"""
|
||||
model_info: Final[dict] = deployment.model_info.model_dump(exclude_none=True) # mutable-ok: built in place
|
||||
for field in CustomPricingLiteLLMParams.model_fields:
|
||||
field_value = deployment.litellm_params.get(field)
|
||||
if field_value is not None:
|
||||
model_info[field] = field_value
|
||||
_copy_custom_pricing_fields(
|
||||
model_info=model_info,
|
||||
litellm_params=deployment.litellm_params,
|
||||
)
|
||||
if model_info.get("input_cost_per_token") is not None:
|
||||
Router._inherit_builtin_cache_pricing(
|
||||
model_info=model_info,
|
||||
|
|
@ -9206,10 +9237,16 @@ class Router:
|
|||
"""
|
||||
if classify_strategy_router_model(model) is not None:
|
||||
model_info = { # mutable-ok: filtered copy of the caller's entry, handed straight to register_model
|
||||
k: v for k, v in model_info.items() if k not in CustomPricingLiteLLMParams.model_fields
|
||||
k: v for k, v in model_info.items() if not _is_custom_pricing_field(k)
|
||||
}
|
||||
|
||||
if model_id is not None:
|
||||
existing_model_info = litellm.model_cost.get(model_id)
|
||||
if existing_model_info is not None:
|
||||
for field in tuple(existing_model_info):
|
||||
if _is_custom_pricing_field(field) and field not in model_info:
|
||||
existing_model_info.pop(field, None)
|
||||
|
||||
litellm.register_model(model_cost={model_id: model_info}, persist_across_reloads=False)
|
||||
|
||||
## OLD MODEL REGISTRATION ## Kept to prevent breaking changes
|
||||
|
|
|
|||
|
|
@ -0,0 +1,418 @@
|
|||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
_get_token_base_cost,
|
||||
)
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tier_field", "tier_value", "rate_index"),
|
||||
[
|
||||
(
|
||||
"output_cost_per_token_above_32k_tokens",
|
||||
18e-6,
|
||||
1,
|
||||
),
|
||||
(
|
||||
"cache_read_input_token_cost_above_32k_tokens",
|
||||
4e-6,
|
||||
4,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_standalone_threshold_field_selects_tier(
|
||||
tier_field: str,
|
||||
tier_value: float,
|
||||
rate_index: int,
|
||||
) -> None:
|
||||
"""Output and cache fields can independently define a tier."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 5e-7,
|
||||
tier_field: tier_value,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=40_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=41_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
assert rates[rate_index] == pytest.approx(tier_value)
|
||||
|
||||
|
||||
def test_cost_types_select_thresholds_independently() -> None:
|
||||
"""An inactive input tier must not suppress an active output tier."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"input_cost_per_token_above_128k_tokens": 9e-6,
|
||||
"output_cost_per_token_above_32k_tokens": 18e-6,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=40_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=41_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
assert rates[0] == pytest.approx(1e-6)
|
||||
assert rates[1] == pytest.approx(18e-6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 1e-6,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
},
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 1e-6,
|
||||
},
|
||||
],
|
||||
ids=["standard-first", "priority-first"],
|
||||
)
|
||||
def test_equal_threshold_matching_tier_wins_for_output(model_info):
|
||||
"""At an equal threshold the matching service-tier rate must win regardless of the
|
||||
order the standard and tier-qualified keys appear in model_info."""
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(model_info=model_info, usage=usage, service_tier="priority")
|
||||
|
||||
assert rates[1] == pytest.approx(1.5e-6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"cache_read_input_token_cost": 2e-7,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 3e-7,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 4e-7,
|
||||
},
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"cache_read_input_token_cost": 2e-7,
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": 4e-7,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 3e-7,
|
||||
},
|
||||
],
|
||||
ids=["standard-first", "priority-first"],
|
||||
)
|
||||
def test_equal_threshold_matching_tier_wins_for_cache_read(model_info):
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(model_info=model_info, usage=usage, service_tier="priority")
|
||||
|
||||
assert rates[4] == pytest.approx(4e-7)
|
||||
|
||||
|
||||
def test_equal_threshold_default_tier_keeps_standard():
|
||||
"""service_tier=None must ignore the tier-qualified key and keep the standard rate."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 1e-6,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(model_info=model_info, usage=usage)
|
||||
|
||||
assert rates[1] == pytest.approx(1e-6)
|
||||
|
||||
|
||||
def test_equal_threshold_wrong_tier_keeps_standard():
|
||||
"""A priority-qualified key must not win under a different service tier."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 1e-6,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier="flex",
|
||||
)
|
||||
|
||||
assert rates[1] == pytest.approx(1e-6)
|
||||
|
||||
|
||||
def test_unequal_threshold_highest_standard_still_wins():
|
||||
"""A higher standard threshold governs over a lower matching-tier threshold."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 1e-6,
|
||||
"output_cost_per_token_above_128k_tokens_priority": 1.5e-6,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier="priority",
|
||||
)
|
||||
|
||||
assert rates[1] == pytest.approx(1e-6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tier_field", "tier_value", "rate_index"),
|
||||
[
|
||||
("output_cost_per_token_above_200k_tokens_priority", 1.5e-6, 1),
|
||||
("cache_read_input_token_cost_above_200k_tokens_priority", 4e-7, 4),
|
||||
],
|
||||
)
|
||||
def test_fast_service_tier_uses_priority_qualified_threshold(tier_field, tier_value, rate_index):
|
||||
"""fast aliases to priority pricing, so a priority-qualified threshold with no standard
|
||||
sibling must apply to a fast request too."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 1e-6,
|
||||
"cache_read_input_token_cost": 2e-7,
|
||||
tier_field: tier_value,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier="fast",
|
||||
)
|
||||
|
||||
assert rates[rate_index] == pytest.approx(tier_value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "expected"),
|
||||
[
|
||||
("priority", 1.5e-6),
|
||||
("flex", 1e-6),
|
||||
(None, 1e-6),
|
||||
],
|
||||
ids=["priority-guard", "flex-guard", "default-guard"],
|
||||
)
|
||||
def test_fast_alias_guard_tiers_for_priority_only_output_threshold(service_tier, expected):
|
||||
"""priority must keep working under the alias; flex and default must not pick up the
|
||||
priority rate."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 1e-6,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
assert rates[1] == pytest.approx(expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("service_tier", ["priority", "fast"], ids=["priority", "fast"])
|
||||
def test_fast_builtin_standard_plus_priority_threshold_resolves_priority(service_tier):
|
||||
"""The builtin shape (standard sibling present) keeps resolving the priority variant via
|
||||
the existing standard-key alias lookup."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 1e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 1e-6,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
assert rates[1] == pytest.approx(1.5e-6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 1e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 1e-6,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
},
|
||||
{
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 1e-6,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 1e-6,
|
||||
},
|
||||
],
|
||||
ids=["standard-first", "priority-first"],
|
||||
)
|
||||
def test_fast_equal_threshold_matching_tier_wins_both_orders(model_info):
|
||||
"""The Phase D equal-threshold tie-break must hold under the fast alias, independent of
|
||||
insertion order."""
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier="fast",
|
||||
)
|
||||
|
||||
assert rates[1] == pytest.approx(1.5e-6)
|
||||
|
||||
|
||||
def test_fast_service_tier_uppercase_uses_priority_qualified_threshold():
|
||||
"""The scanner must be as case-insensitive as the lookup path."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 1e-6,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier="FAST",
|
||||
)
|
||||
|
||||
assert rates[1] == pytest.approx(1.5e-6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tier_field", "tier_value", "rate_index"),
|
||||
[
|
||||
("output_cost_per_token_above_200k_tokens_priority", 1.5e-6, 1),
|
||||
("cache_read_input_token_cost_above_200k_tokens_priority", 4e-7, 4),
|
||||
],
|
||||
)
|
||||
def test_tier_qualified_threshold_field_selects_tier_for_matching_service_tier(
|
||||
tier_field: str, tier_value: float, rate_index: int
|
||||
) -> None:
|
||||
"""A tier-qualified threshold key with no standard sibling must apply when the request's
|
||||
service tier matches."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 5e-7,
|
||||
tier_field: tier_value,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier="priority",
|
||||
)
|
||||
|
||||
assert rates[rate_index] == pytest.approx(tier_value)
|
||||
|
||||
|
||||
def test_tier_qualified_threshold_is_ignored_under_the_default_tier() -> None:
|
||||
"""service_tier=None must keep billing the flat rate: the priority key is inactive."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 5e-7,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(model_info=model_info, usage=usage)
|
||||
|
||||
assert rates[1] == pytest.approx(2e-6)
|
||||
|
||||
|
||||
def test_tier_qualified_threshold_is_ignored_under_a_different_service_tier() -> None:
|
||||
"""A priority-qualified key must not bill a flex request."""
|
||||
model_info = {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"cache_read_input_token_cost": 5e-7,
|
||||
"output_cost_per_token_above_200k_tokens_priority": 1.5e-6,
|
||||
}
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
)
|
||||
|
||||
rates = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier="flex",
|
||||
)
|
||||
|
||||
assert rates[1] == pytest.approx(2e-6)
|
||||
|
|
@ -7,6 +7,7 @@ from unittest.mock import patch
|
|||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.ptu_pricing import (
|
||||
is_threshold_rate_key,
|
||||
ptu_config_error,
|
||||
ptu_identity_error,
|
||||
CUSTOM_PRICING_FIELDS,
|
||||
|
|
@ -165,6 +166,115 @@ def test_a_rate_the_deployment_declares_itself_is_zeroed_too():
|
|||
assert override[extra] == 0.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"threshold_field",
|
||||
[
|
||||
"input_cost_per_token_above_32k_tokens",
|
||||
"output_cost_per_token_above_32k_tokens",
|
||||
"cache_read_input_token_cost_above_32k_tokens",
|
||||
"cache_creation_input_token_cost_above_32k_tokens",
|
||||
"cache_creation_input_token_cost_above_1hr_above_32k_tokens",
|
||||
],
|
||||
)
|
||||
def test_arbitrary_threshold_rate_the_deployment_declares_is_zeroed_too(threshold_field):
|
||||
"""Threshold rates a deployment can declare but CustomPricingLiteLLMParams does not
|
||||
enumerate (e.g. input_cost_per_token_above_32k_tokens) must be zeroed like any other
|
||||
declared rate, or a PTU deployment bills per token past the threshold on top of the
|
||||
hourly charge."""
|
||||
assert threshold_field not in CUSTOM_PRICING_FIELDS
|
||||
assert threshold_field not in PTU_ZEROED_PRICING_FIELDS
|
||||
|
||||
override = _with_flag(_VALID, declared={threshold_field: 9e-06})
|
||||
|
||||
assert override is not None
|
||||
assert override.get(threshold_field, 9e-06) == 0.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"threshold_field",
|
||||
[
|
||||
"output_cost_per_token_above_32k_tokens_priority",
|
||||
"cache_read_input_token_cost_above_32k_tokens_priority",
|
||||
],
|
||||
)
|
||||
def test_service_tier_qualified_threshold_rate_declared_is_zeroed_too(threshold_field):
|
||||
"""A tier-qualified threshold rate the deployment declares must be zeroed like every
|
||||
other declared rate, or a PTU deployment bills per token past the threshold."""
|
||||
assert threshold_field not in CUSTOM_PRICING_FIELDS
|
||||
assert threshold_field not in PTU_ZEROED_PRICING_FIELDS
|
||||
|
||||
override = _with_flag(_VALID, declared={threshold_field: 9e-06})
|
||||
|
||||
assert override is not None
|
||||
assert override.get(threshold_field, 9e-06) == 0.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"param_field",
|
||||
[
|
||||
"api_key_above_32k_tokens",
|
||||
"secret_above_32k_tokens",
|
||||
"credential_above_32k_tokens",
|
||||
"custom_provider_param_above_32k_tokens",
|
||||
],
|
||||
)
|
||||
def test_threshold_like_non_pricing_params_are_left_alone(param_field):
|
||||
"""The threshold matcher must only cover the cost calculator's pricing base keys: a
|
||||
deployment param that merely ends in _above_<N>k_tokens is not a charge and must keep
|
||||
its value."""
|
||||
override = _with_flag(_VALID, declared={param_field: "keep-me"})
|
||||
|
||||
assert override is not None
|
||||
assert param_field not in override
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field",
|
||||
[
|
||||
"input_cost_per_token_above_32k_tokens",
|
||||
"output_cost_per_token_above_64k_tokens",
|
||||
"cache_read_input_token_cost_above_128k_tokens",
|
||||
"cache_creation_input_token_cost_above_200k_tokens",
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens",
|
||||
],
|
||||
)
|
||||
def test_is_threshold_rate_key_accepts_supported_threshold_rates(field):
|
||||
assert is_threshold_rate_key(field)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field",
|
||||
[
|
||||
"input_cost_per_token_above_32k_tokens_priority",
|
||||
"output_cost_per_token_above_32k_tokens_flex",
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority",
|
||||
"cache_creation_input_token_cost_above_32k_tokens_ultrafast",
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens_priority",
|
||||
],
|
||||
)
|
||||
def test_is_threshold_rate_key_accepts_service_tier_qualified_threshold_rates(field):
|
||||
assert is_threshold_rate_key(field)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field",
|
||||
[
|
||||
"api_key_above_32k_tokens",
|
||||
"secret_above_32k_tokens",
|
||||
"credential_above_32k_tokens",
|
||||
"custom_provider_param_above_32k_tokens",
|
||||
"api_key_above_32k_tokens_priority",
|
||||
"secret_above_32k_tokens_flex",
|
||||
"credential_above_200k_tokens_ultrafast",
|
||||
"custom_provider_param_above_32k_tokens_priority",
|
||||
"input_cost_per_token",
|
||||
"tiered_pricing",
|
||||
],
|
||||
)
|
||||
def test_is_threshold_rate_key_rejects_non_pricing_fields(field):
|
||||
assert not is_threshold_rate_key(field)
|
||||
|
||||
|
||||
def test_a_setting_that_is_not_a_charge_is_left_alone():
|
||||
"""CustomPricingLiteLLMParams also carries configuration, and zeroing one of those
|
||||
would break the deployment rather than stop a charge."""
|
||||
|
|
|
|||
|
|
@ -794,6 +794,22 @@ class TestPtuDeploymentsAreNotBilledPerToken:
|
|||
assert exc.value.status_code == 400
|
||||
assert "tiered_pricing" in str(exc.value.detail)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field",
|
||||
[
|
||||
"input_cost_per_token_above_32k_tokens",
|
||||
"output_cost_per_token_above_32k_tokens",
|
||||
"input_cost_per_token_above_32k_tokens_priority",
|
||||
],
|
||||
)
|
||||
def test_an_arbitrary_threshold_rate_the_caller_supplies_is_refused(self, field):
|
||||
"""The router lets deployments declare arbitrary above-threshold rates; those bill a
|
||||
PTU deployment past the threshold just as surely as a flat rate, so the refusal must
|
||||
cover them too."""
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
self._zeroed(model_info=self.PTU, supplied={field: 9e-06})
|
||||
assert exc.value.status_code == 400
|
||||
assert field in str(exc.value.detail)
|
||||
def test_a_search_context_price_the_caller_supplies_is_refused(self):
|
||||
"""The rates sit in a table keyed by context size, so a guard that only reads numbers
|
||||
lets a per-request charge onto a deployment its reserved capacity already pays for."""
|
||||
|
|
@ -880,6 +896,19 @@ class TestPtuDeploymentsAreNotBilledPerToken:
|
|||
assert zeroed["input_cost_per_second"] == 0
|
||||
assert zeroed["input_cost_per_token"] == 0
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field",
|
||||
[
|
||||
"input_cost_per_token_above_32k_tokens",
|
||||
"output_cost_per_token_above_32k_tokens",
|
||||
"output_cost_per_token_above_32k_tokens_priority",
|
||||
],
|
||||
)
|
||||
def test_an_arbitrary_threshold_rate_already_on_the_row_is_zeroed(self, field):
|
||||
"""A row priced through a path this rule does not cover must heal on its next save."""
|
||||
zeroed = self._zeroed(model_info={**self.PTU, field: 9e-06}, litellm_params={})
|
||||
assert zeroed[field] == 0
|
||||
assert zeroed["input_cost_per_token"] == 0
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_refused_price_does_not_leave_the_team_changed(self):
|
||||
"""The team ACL write autocommits, so the refusal has to run before it. Otherwise a
|
||||
|
|
@ -944,6 +973,34 @@ class TestPtuDeploymentsAreNotBilledPerToken:
|
|||
assert priced.litellm_params.get("regional_processing_uplift_multiplier_eu") == 1.15
|
||||
assert priced.litellm_params.get("input_cost_per_token") == 0
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"param",
|
||||
[
|
||||
"api_key_above_32k_tokens",
|
||||
"secret_above_32k_tokens",
|
||||
"credential_above_32k_tokens",
|
||||
"custom_provider_param_above_32k_tokens",
|
||||
"api_key_above_32k_tokens_priority",
|
||||
"custom_provider_param_above_32k_tokens_flex",
|
||||
],
|
||||
)
|
||||
def test_a_threshold_like_setting_that_is_not_a_price_is_left_alone(self, param):
|
||||
"""Only the cost calculator's base keys qualify as threshold rates: a deployment param
|
||||
that merely ends in _above_<N>k_tokens is not a charge and must keep its value."""
|
||||
priced = _ptu_priced_deployment(
|
||||
Deployment(
|
||||
model_name="settings",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-4o-mini", **{param: "keep-me"}),
|
||||
model_info=ModelInfo(
|
||||
id="dep-settings",
|
||||
team_id="t",
|
||||
ptu_effective_from=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc),
|
||||
**self.PTU,
|
||||
),
|
||||
)
|
||||
)
|
||||
assert priced.litellm_params.get(param) == "keep-me"
|
||||
assert priced.litellm_params.get("input_cost_per_token") == 0
|
||||
def test_removing_ptu_config_releases_every_rate_it_zeroed(self):
|
||||
"""The zeroing covers any stored rate, so a release that only spans the mirrored fields
|
||||
leaves a per-second deployment billing nothing for that dimension forever."""
|
||||
|
|
@ -976,6 +1033,73 @@ class TestPtuDeploymentsAreNotBilledPerToken:
|
|||
)
|
||||
assert "input_cost_per_second" not in json.loads(off["litellm_params"])
|
||||
|
||||
def test_removing_ptu_config_releases_a_zeroed_threshold_rate(self):
|
||||
"""The zeroing spans threshold rates too, so a release that only spans the enumeration
|
||||
sets would leave one billing nothing past the threshold forever."""
|
||||
on = update_db_model(
|
||||
db_model=Deployment(
|
||||
model_name="threshold",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-4o-mini", input_cost_per_token_above_32k_tokens=9e-06),
|
||||
model_info=ModelInfo(id="dep-thr", team_id="t"),
|
||||
),
|
||||
updated_patch=updateDeployment(
|
||||
model_info=ModelInfo(
|
||||
id="dep-thr",
|
||||
team_id="t",
|
||||
ptu_effective_from=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc),
|
||||
**self.PTU,
|
||||
)
|
||||
),
|
||||
)
|
||||
assert json.loads(on["litellm_params"])["input_cost_per_token_above_32k_tokens"] == 0
|
||||
|
||||
off = update_db_model(
|
||||
db_model=Deployment(
|
||||
model_name="threshold",
|
||||
litellm_params=LiteLLM_Params(**json.loads(on["litellm_params"])),
|
||||
model_info=ModelInfo(**json.loads(on["model_info"])),
|
||||
),
|
||||
updated_patch=updateDeployment(
|
||||
model_info=ModelInfo(id="dep-thr", ptu_count=None, cost_per_ptu_per_hour=None)
|
||||
),
|
||||
)
|
||||
assert "input_cost_per_token_above_32k_tokens" not in json.loads(off["litellm_params"])
|
||||
|
||||
def test_removing_ptu_config_releases_a_zeroed_tier_qualified_threshold_rate(self):
|
||||
"""The zeroing spans tier-qualified threshold rates too, so a release that only
|
||||
spans the enumeration sets would leave one billing nothing past the threshold
|
||||
forever."""
|
||||
on = update_db_model(
|
||||
db_model=Deployment(
|
||||
model_name="tier-threshold",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="openai/gpt-4o-mini", output_cost_per_token_above_32k_tokens_priority=9e-06
|
||||
),
|
||||
model_info=ModelInfo(id="dep-tier-thr", team_id="t"),
|
||||
),
|
||||
updated_patch=updateDeployment(
|
||||
model_info=ModelInfo(
|
||||
id="dep-tier-thr",
|
||||
team_id="t",
|
||||
ptu_effective_from=datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc),
|
||||
**self.PTU,
|
||||
)
|
||||
),
|
||||
)
|
||||
assert json.loads(on["litellm_params"])["output_cost_per_token_above_32k_tokens_priority"] == 0
|
||||
|
||||
off = update_db_model(
|
||||
db_model=Deployment(
|
||||
model_name="tier-threshold",
|
||||
litellm_params=LiteLLM_Params(**json.loads(on["litellm_params"])),
|
||||
model_info=ModelInfo(**json.loads(on["model_info"])),
|
||||
),
|
||||
updated_patch=updateDeployment(
|
||||
model_info=ModelInfo(id="dep-tier-thr", ptu_count=None, cost_per_ptu_per_hour=None)
|
||||
),
|
||||
)
|
||||
assert "output_cost_per_token_above_32k_tokens_priority" not in json.loads(off["litellm_params"])
|
||||
|
||||
def test_removing_ptu_config_releases_a_zeroed_search_context_table(self):
|
||||
"""The all-zero table exists only to stop the double charge, so a deployment taken off PTU
|
||||
has to give it up or it keeps serving grounded requests for free forever."""
|
||||
|
|
|
|||
|
|
@ -19,6 +19,10 @@ import pytest
|
|||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.litellm_core_utils.ptu_pricing import ptu_config_error
|
||||
from litellm.router import (
|
||||
_copy_custom_pricing_fields,
|
||||
_is_custom_pricing_field,
|
||||
)
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
from litellm.utils import (
|
||||
_invalidate_model_cost_lowercase_map,
|
||||
|
|
@ -60,6 +64,117 @@ def _restore_model_cost_entries(original_entries):
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_is_custom_pricing_field_recognizes_declared_and_arbitrary_threshold_fields():
|
||||
assert _is_custom_pricing_field("input_cost_per_token")
|
||||
assert _is_custom_pricing_field("input_cost_per_token_above_32k_tokens")
|
||||
assert not _is_custom_pricing_field("api_key")
|
||||
assert not _is_custom_pricing_field("input_cost_per_token_above_32k_tokens_extra")
|
||||
|
||||
|
||||
def test_is_custom_pricing_field_recognizes_tier_qualified_threshold_fields():
|
||||
assert _is_custom_pricing_field("input_cost_per_token_above_200k_tokens_priority")
|
||||
assert _is_custom_pricing_field("output_cost_per_token_above_200k_tokens_flex")
|
||||
assert _is_custom_pricing_field("cache_read_input_token_cost_above_200k_tokens_priority")
|
||||
assert _is_custom_pricing_field("cache_creation_input_token_cost_above_1hr_above_200k_tokens_ultrafast")
|
||||
assert not _is_custom_pricing_field("api_key_above_32k_tokens_priority")
|
||||
assert not _is_custom_pricing_field("secret_above_32k_tokens_flex")
|
||||
assert not _is_custom_pricing_field("credential_above_200k_tokens_ultrafast")
|
||||
assert not _is_custom_pricing_field("custom_provider_param_above_32k_tokens_priority")
|
||||
|
||||
|
||||
def test_custom_tier_qualified_threshold_key_is_registered():
|
||||
"""A deployment declaring only a tier-qualified threshold rate (e.g.
|
||||
output_cost_per_token_above_200k_tokens_priority, which is not enumerated in
|
||||
CustomPricingLiteLLMParams) must keep it in its model_cost entry so matching-tier
|
||||
requests bill it."""
|
||||
model_id = "custom-tier-only-threshold"
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "tier-only-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "fake-key",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token_above_32k_tokens_priority": 1.5e-6,
|
||||
},
|
||||
"model_info": {"id": model_id},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
registered = litellm.model_cost[model_id]
|
||||
assert registered["output_cost_per_token_above_32k_tokens_priority"] == 1.5e-6
|
||||
|
||||
def test_copy_custom_pricing_fields_preserves_declared_and_arbitrary_threshold_fields():
|
||||
"""Copy supported pricing fields without copying unrelated LiteLLM params."""
|
||||
model_info = {}
|
||||
litellm_params = LiteLLM_Params(
|
||||
model="openai/gpt-4o-mini",
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=1e-6,
|
||||
input_cost_per_token_above_32k_tokens=9e-6,
|
||||
)
|
||||
|
||||
_copy_custom_pricing_fields(
|
||||
model_info=model_info,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
assert model_info == {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"input_cost_per_token_above_32k_tokens": 9e-6,
|
||||
}
|
||||
|
||||
|
||||
def test_is_custom_pricing_field_accepts_supported_threshold_base_keys():
|
||||
"""Every threshold base key the cost calculator supports stays accepted."""
|
||||
for field in (
|
||||
"input_cost_per_token_above_32k_tokens",
|
||||
"output_cost_per_token_above_32k_tokens",
|
||||
"cache_read_input_token_cost_above_32k_tokens",
|
||||
"cache_creation_input_token_cost_above_32k_tokens",
|
||||
"cache_creation_input_token_cost_above_1hr_above_32k_tokens",
|
||||
):
|
||||
assert _is_custom_pricing_field(field), field
|
||||
|
||||
|
||||
def test_is_custom_pricing_field_rejects_non_pricing_threshold_like_params():
|
||||
"""Deployment params that merely look like *_above_<N>_tokens are not pricing."""
|
||||
for field in (
|
||||
"api_key_above_32k_tokens",
|
||||
"secret_above_32k_tokens",
|
||||
"credential_above_32k_tokens",
|
||||
"custom_provider_param_above_32k_tokens",
|
||||
):
|
||||
assert not _is_custom_pricing_field(field), field
|
||||
|
||||
|
||||
def test_copy_custom_pricing_fields_ignores_non_pricing_threshold_like_params():
|
||||
"""Only supported pricing fields are copied; threshold-like deployment params stay out of model_info."""
|
||||
model_info = {}
|
||||
litellm_params = LiteLLM_Params(
|
||||
model="openai/gpt-4o-mini",
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=1e-6,
|
||||
input_cost_per_token_above_32k_tokens=9e-6,
|
||||
api_key_above_32k_tokens="leaked",
|
||||
secret_above_32k_tokens="leaked",
|
||||
credential_above_32k_tokens="leaked",
|
||||
custom_provider_param_above_32k_tokens="leaked",
|
||||
)
|
||||
|
||||
_copy_custom_pricing_fields(
|
||||
model_info=model_info,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
assert model_info == {
|
||||
"input_cost_per_token": 1e-6,
|
||||
"input_cost_per_token_above_32k_tokens": 9e-6,
|
||||
}
|
||||
|
||||
|
||||
def test_should_not_pollute_shared_key_with_zero_cost_pricing():
|
||||
"""
|
||||
When deployment A has input_cost_per_token=0 and deployment B has no
|
||||
|
|
@ -1426,6 +1541,270 @@ def test_replay_live_router_model_cost_rebuilds_every_live_router():
|
|||
litellm.model_cost = saved_model_cost
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
def test_router_registers_arbitrary_above_threshold_pricing_from_litellm_params():
|
||||
"""Regression for #34378.
|
||||
|
||||
Arbitrary *_above_<N>_tokens pricing fields accepted in litellm_params
|
||||
must be registered under the deployment model ID.
|
||||
"""
|
||||
backend_model = "openai/gpt-4o-mini"
|
||||
model_id = "router-custom-above-32k-pricing-34378"
|
||||
shared_keys = ("gpt-4o-mini", backend_model)
|
||||
|
||||
original_entries = {
|
||||
key: copy.deepcopy(litellm.model_cost.get(key))
|
||||
for key in (*shared_keys, model_id)
|
||||
}
|
||||
|
||||
try:
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "custom-above-32k-model",
|
||||
"litellm_params": {
|
||||
"model": backend_model,
|
||||
"api_key": "fake-key",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"input_cost_per_token_above_32k_tokens": 9e-6,
|
||||
"output_cost_per_token_above_32k_tokens": 18e-6,
|
||||
"cache_read_input_token_cost_above_32k_tokens": 4e-6,
|
||||
},
|
||||
"model_info": {
|
||||
"id": model_id,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
registered = litellm.model_cost.get(model_id)
|
||||
assert registered is not None
|
||||
|
||||
assert registered["input_cost_per_token"] == 1e-6
|
||||
assert registered["output_cost_per_token"] == 2e-6
|
||||
assert registered["input_cost_per_token_above_32k_tokens"] == 9e-6
|
||||
assert registered["output_cost_per_token_above_32k_tokens"] == 18e-6
|
||||
assert registered["cache_read_input_token_cost_above_32k_tokens"] == 4e-6
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
generic_cost_per_token,
|
||||
)
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
prompt_tokens = 40_000
|
||||
completion_tokens = 1_000
|
||||
usage = Usage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model=model_id,
|
||||
usage=usage,
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(9e-6 * prompt_tokens)
|
||||
assert completion_cost == pytest.approx(18e-6 * completion_tokens)
|
||||
finally:
|
||||
_restore_model_cost_entries(original_entries)
|
||||
|
||||
|
||||
def test_router_registration_never_registers_non_pricing_threshold_like_params():
|
||||
"""Non-pricing *_above_<N>_tokens deployment params must not reach litellm.model_cost."""
|
||||
backend_model = "openai/gpt-4o-mini"
|
||||
model_id = "router-rejects-non-pricing-above-32k-34504"
|
||||
shared_keys = ("gpt-4o-mini", backend_model)
|
||||
|
||||
original_entries = {
|
||||
key: copy.deepcopy(litellm.model_cost.get(key))
|
||||
for key in (*shared_keys, model_id)
|
||||
}
|
||||
|
||||
try:
|
||||
Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "custom-above-32k-model",
|
||||
"litellm_params": {
|
||||
"model": backend_model,
|
||||
"api_key": "fake-key",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"input_cost_per_token_above_32k_tokens": 9e-6,
|
||||
"api_key_above_32k_tokens": "leaked",
|
||||
"secret_above_32k_tokens": "leaked",
|
||||
"credential_above_32k_tokens": "leaked",
|
||||
"custom_provider_param_above_32k_tokens": "leaked",
|
||||
},
|
||||
"model_info": {
|
||||
"id": model_id,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
registered = litellm.model_cost.get(model_id)
|
||||
assert registered is not None
|
||||
|
||||
assert registered["input_cost_per_token"] == 1e-6
|
||||
assert registered["output_cost_per_token"] == 2e-6
|
||||
assert registered["input_cost_per_token_above_32k_tokens"] == 9e-6
|
||||
for junk in (
|
||||
"api_key_above_32k_tokens",
|
||||
"secret_above_32k_tokens",
|
||||
"credential_above_32k_tokens",
|
||||
"custom_provider_param_above_32k_tokens",
|
||||
):
|
||||
assert junk not in registered, junk
|
||||
finally:
|
||||
_restore_model_cost_entries(original_entries)
|
||||
|
||||
|
||||
def test_add_deployment_registers_arbitrary_above_threshold_pricing():
|
||||
"""Dynamic add_deployment must preserve arbitrary pricing thresholds."""
|
||||
backend_model = "openai/gpt-4o-mini"
|
||||
model_id = "add-deployment-custom-above-32k-pricing-34378"
|
||||
shared_keys = ("gpt-4o-mini", backend_model)
|
||||
|
||||
original_entries = {
|
||||
key: copy.deepcopy(litellm.model_cost.get(key))
|
||||
for key in (*shared_keys, model_id)
|
||||
}
|
||||
|
||||
try:
|
||||
router = Router(model_list=[])
|
||||
router.add_deployment(
|
||||
deployment=Deployment(
|
||||
model_name="dynamic-custom-above-32k-model",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model=backend_model,
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=1e-6,
|
||||
output_cost_per_token=2e-6,
|
||||
input_cost_per_token_above_32k_tokens=9e-6,
|
||||
output_cost_per_token_above_32k_tokens=18e-6,
|
||||
cache_read_input_token_cost_above_32k_tokens=4e-6,
|
||||
),
|
||||
model_info=ModelInfo(id=model_id),
|
||||
)
|
||||
)
|
||||
|
||||
registered = litellm.model_cost.get(model_id)
|
||||
assert registered is not None
|
||||
assert registered["input_cost_per_token_above_32k_tokens"] == 9e-6
|
||||
assert registered["output_cost_per_token_above_32k_tokens"] == 18e-6
|
||||
assert registered["cache_read_input_token_cost_above_32k_tokens"] == 4e-6
|
||||
|
||||
for shared_key in shared_keys:
|
||||
shared_entry = litellm.model_cost.get(shared_key) or {}
|
||||
assert "input_cost_per_token_above_32k_tokens" not in shared_entry
|
||||
assert "output_cost_per_token_above_32k_tokens" not in shared_entry
|
||||
assert "cache_read_input_token_cost_above_32k_tokens" not in shared_entry
|
||||
finally:
|
||||
_restore_model_cost_entries(original_entries)
|
||||
|
||||
|
||||
def test_upsert_deployment_removes_withdrawn_arbitrary_above_threshold_pricing():
|
||||
"""Updating a deployment must not retain arbitrary pricing tiers that were removed."""
|
||||
backend_model = "openai/gpt-4o-mini"
|
||||
model_id = "upsert-withdrawn-above-32k-pricing-34378"
|
||||
shared_keys = ("gpt-4o-mini", backend_model)
|
||||
|
||||
original_entries = {
|
||||
key: copy.deepcopy(litellm.model_cost.get(key))
|
||||
for key in (*shared_keys, model_id)
|
||||
}
|
||||
|
||||
try:
|
||||
router = Router(model_list=[])
|
||||
|
||||
router.add_deployment(
|
||||
deployment=Deployment(
|
||||
model_name="upsert-custom-above-32k-model",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model=backend_model,
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=1e-6,
|
||||
output_cost_per_token=2e-6,
|
||||
input_cost_per_token_above_32k_tokens=9e-6,
|
||||
),
|
||||
model_info=ModelInfo(id=model_id),
|
||||
)
|
||||
)
|
||||
|
||||
assert litellm.model_cost[model_id]["input_cost_per_token_above_32k_tokens"] == 9e-6
|
||||
|
||||
router.upsert_deployment(
|
||||
deployment=Deployment(
|
||||
model_name="upsert-custom-above-32k-model",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model=backend_model,
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=1e-6,
|
||||
output_cost_per_token=2e-6,
|
||||
),
|
||||
model_info=ModelInfo(id=model_id),
|
||||
)
|
||||
)
|
||||
|
||||
registered = litellm.model_cost[model_id]
|
||||
assert registered["input_cost_per_token"] == 1e-6
|
||||
assert "input_cost_per_token_above_32k_tokens" not in registered
|
||||
finally:
|
||||
_restore_model_cost_entries(original_entries)
|
||||
|
||||
|
||||
def test_price_data_reload_preserves_arbitrary_above_threshold_pricing(monkeypatch):
|
||||
"""Arbitrary pricing thresholds must survive Router cost-map replay."""
|
||||
from litellm import utils as litellm_utils
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm_utils,
|
||||
"_runtime_registered_model_cost",
|
||||
dict(litellm_utils._runtime_registered_model_cost),
|
||||
)
|
||||
|
||||
model_id = "reload-custom-above-32k-pricing-34378"
|
||||
saved_model_cost = copy.deepcopy(litellm.model_cost)
|
||||
|
||||
try:
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "reload-custom-above-32k-model",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "fake-key",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 2e-6,
|
||||
"input_cost_per_token_above_32k_tokens": 9e-6,
|
||||
"output_cost_per_token_above_32k_tokens": 18e-6,
|
||||
"cache_read_input_token_cost_above_32k_tokens": 4e-6,
|
||||
},
|
||||
"model_info": {"id": model_id},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
before = litellm.model_cost[model_id]
|
||||
assert before["input_cost_per_token_above_32k_tokens"] == 9e-6
|
||||
assert before["output_cost_per_token_above_32k_tokens"] == 18e-6
|
||||
assert before["cache_read_input_token_cost_above_32k_tokens"] == 4e-6
|
||||
|
||||
_simulate_price_data_reload(
|
||||
{"gpt-4o": {"litellm_provider": "openai", "mode": "chat"}},
|
||||
)
|
||||
|
||||
rebuilt = litellm.model_cost[model_id]
|
||||
assert rebuilt["input_cost_per_token_above_32k_tokens"] == 9e-6
|
||||
assert rebuilt["output_cost_per_token_above_32k_tokens"] == 18e-6
|
||||
assert rebuilt["cache_read_input_token_cost_above_32k_tokens"] == 4e-6
|
||||
assert router.model_list
|
||||
finally:
|
||||
_restore_model_cost_entries(saved_model_cost)
|
||||
|
||||
|
||||
def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch):
|
||||
"""
|
||||
|
|
@ -1453,6 +1832,7 @@ def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch):
|
|||
"complexity_router_default_model": "paid-model",
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
"input_cost_per_token_above_32k_tokens": 9e-6,
|
||||
"complexity_router_config": {"tiers": {"simple": "paid-model"}},
|
||||
},
|
||||
"model_info": {"id": "strategy-alias-id", "max_input_tokens": 128000},
|
||||
|
|
@ -1471,6 +1851,7 @@ def test_strategy_router_alias_pricing_never_enters_model_cost(monkeypatch):
|
|||
assert entry["max_input_tokens"] == 128000
|
||||
assert "input_cost_per_token" not in entry
|
||||
assert "output_cost_per_token" not in entry
|
||||
assert "input_cost_per_token_above_32k_tokens" not in entry
|
||||
|
||||
_assert_alias_unpriced()
|
||||
|
||||
|
|
@ -1583,6 +1964,24 @@ def test_a_config_ptu_deployment_bills_nothing_per_token():
|
|||
assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token"] == 0.0
|
||||
|
||||
|
||||
def test_a_config_ptu_deployment_zeroes_custom_threshold_rates():
|
||||
"""A custom threshold rate the deployment declares (e.g.
|
||||
input_cost_per_token_above_32k_tokens) must be zeroed like every other declared rate,
|
||||
or the PTU deployment bills per token past the threshold on top of the hourly charge."""
|
||||
router = _ptu_router(
|
||||
litellm_params={
|
||||
"input_cost_per_token": 5e-06,
|
||||
"input_cost_per_token_above_32k_tokens": 9e-06,
|
||||
}
|
||||
)
|
||||
entry = router.model_list[0]
|
||||
|
||||
assert entry["litellm_params"]["input_cost_per_token"] == 0.0
|
||||
assert entry["litellm_params"]["input_cost_per_token_above_32k_tokens"] == 0.0
|
||||
assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token"] == 0.0
|
||||
assert litellm.model_cost[entry["model_info"]["id"]]["input_cost_per_token_above_32k_tokens"] == 0.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend",
|
||||
["anthropic/claude-sonnet-4-5-20250929", "azure/gpt-4o", "gemini/gemini-2.5-flash"],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue