scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix(cost_calculator): recognize the ultrafast service tier in cost calculation

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-08-18 20:43:55 +00:00
parent ceeab01b0d
commit ac17352594
4 changed files with 102 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -42,14 +42,18 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency)
# Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per
# request in the cost-calc path, so the f-strings are built once here instead
# of being rebuilt for every model_info key on every call.
_SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple(f"_{st.value}" for st in ServiceTier)
# of being rebuilt for every model_info key on every call. Longest-first so a
# substring match resolves "_ultrafast" before "_fast".
_SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple(
sorted((f"_{st.value}" for st in ServiceTier), key=len, reverse=True)
)
_SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
{
ServiceTier.FLEX.value: ServiceTier.FLEX.value,
ServiceTier.PRIORITY.value: ServiceTier.PRIORITY.value,
ServiceTier.FAST.value: ServiceTier.PRIORITY.value,
ServiceTier.ULTRAFAST.value: ServiceTier.ULTRAFAST.value,
}
)
@ -191,7 +195,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str:
Args:
base_key: The base cost key (e.g., "input_cost_per_token")
service_tier: The service tier ("flex", "priority", "fast", or None for standard)
service_tier: The service tier ("flex", "priority", "fast", "ultrafast", or None for standard)
Returns:
str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")

9
litellm/types/utils.py
View file

@ -196,6 +196,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_token: Required[float | None]
input_cost_per_token_flex: float | None # OpenAI flex service tier pricing
input_cost_per_token_priority: float | None # OpenAI priority service tier pricing
input_cost_per_token_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_creation_input_token_cost: float | None
cache_creation_input_token_cost_above_200k_tokens: float | None
cache_creation_input_token_cost_above_272k_tokens: float | None
@ -204,9 +205,11 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_creation_input_token_cost_above_1hr: float | None
cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_read_input_token_cost: float | None
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_read_input_token_cost_above_200k_tokens: float | None
cache_read_input_token_cost_above_200k_tokens_priority: float | None
cache_read_input_token_cost_above_272k_tokens: float | None
@ -238,6 +241,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
output_cost_per_token: Required[float | None]
output_cost_per_token_flex: float | None # OpenAI flex service tier pricing
output_cost_per_token_priority: float | None # OpenAI priority service tier pricing
output_cost_per_token_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
regional_processing_uplift_multiplier_eu: (
float | None
) # OpenAI EU data-residency uplift multiplier applied to all token costs (e.g. 1.10 = +10%)
@ -3291,6 +3295,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
# This allows any model_info parameter to be set in litellm_params
input_cost_per_token_flex: float | None = None
input_cost_per_token_priority: float | None = None
input_cost_per_token_ultrafast: float | None = None
cache_creation_input_token_cost_above_1hr: float | None = None
cache_creation_input_token_cost_above_200k_tokens: float | None = None
cache_creation_input_token_cost_above_272k_tokens: float | None = None
@ -3298,9 +3303,11 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
cache_creation_input_token_cost_above_272k_tokens_flex: float | None = None
cache_creation_input_token_cost_flex: float | None = None
cache_creation_input_token_cost_priority: float | None = None
cache_creation_input_token_cost_ultrafast: float | None = None
cache_creation_input_audio_token_cost: float | None = None
cache_read_input_token_cost_flex: float | None = None
cache_read_input_token_cost_priority: float | None = None
cache_read_input_token_cost_ultrafast: float | None = None
cache_read_input_token_cost_above_200k_tokens: float | None = None
cache_read_input_token_cost_above_200k_tokens_priority: float | None = None
cache_read_input_token_cost_above_272k_tokens_priority: float | None = None
@ -3327,6 +3334,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
output_cost_per_token_batches: float | None = None
output_cost_per_token_flex: float | None = None
output_cost_per_token_priority: float | None = None
output_cost_per_token_ultrafast: float | None = None
output_cost_per_audio_token: float | None = None
output_cost_per_token_above_128k_tokens: float | None = None
output_cost_per_token_above_200k_tokens: float | None = None
@ -3994,6 +4002,7 @@ class ServiceTier(Enum):
FLEX = "flex"
PRIORITY = "priority"
FAST = "fast"
ULTRAFAST = "ultrafast"
class DataResidency(Enum):

6
litellm/utils.py
View file

@ -5578,6 +5578,7 @@ def _get_model_info_helper(
input_cost_per_token=_input_cost_per_token,
input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None),
input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None),
input_cost_per_token_ultrafast=_model_info.get("input_cost_per_token_ultrafast", None),
cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None),
cache_creation_input_token_cost_above_200k_tokens=_model_info.get(
"cache_creation_input_token_cost_above_200k_tokens", None
@ -5595,6 +5596,9 @@ def _get_model_info_helper(
cache_creation_input_token_cost_priority=_model_info.get(
"cache_creation_input_token_cost_priority", None
),
cache_creation_input_token_cost_ultrafast=_model_info.get(
"cache_creation_input_token_cost_ultrafast", None
),
cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None),
prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None),
cache_read_input_token_cost_above_200k_tokens=_model_info.get(
@ -5617,6 +5621,7 @@ def _get_model_info_helper(
),
cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None),
cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None),
cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None),
cache_creation_input_token_cost_above_1hr=_model_info.get(
"cache_creation_input_token_cost_above_1hr", None
),
@ -5647,6 +5652,7 @@ def _get_model_info_helper(
output_cost_per_token=_output_cost_per_token,
output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None),
output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None),
output_cost_per_token_ultrafast=_model_info.get("output_cost_per_token_ultrafast", None),
regional_processing_uplift_multiplier_eu=_model_info.get(
"regional_processing_uplift_multiplier_eu", None
),

81
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -1781,6 +1781,81 @@ def test_service_tier_fallback_pricing():
), f"Standard completion cost mismatch: {std_cost[1]} vs {expected_standard_completion}"
def test_service_tier_ultrafast_pricing():
"""An ultrafast request bills the *_ultrafast rates for all token types.
Regression for the ultrafast service tier being absent from ServiceTier:
the cost-key lookup silently returned the standard keys, undercounting
every ultrafast request.
"""
cached_tokens = 200
cache_write_tokens = 300
text_tokens = 500
usage = Usage(
prompt_tokens=text_tokens + cached_tokens + cache_write_tokens,
completion_tokens=400,
total_tokens=text_tokens + cached_tokens + cache_write_tokens + 400,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=cached_tokens, cache_write_tokens=cache_write_tokens
),
)
model_info: ModelInfo = {
"key": "gpt-5.6-sol",
"input_cost_per_token": 5e-06,
"output_cost_per_token": 3e-05,
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token_ultrafast": 5e-05,
"output_cost_per_token_ultrafast": 3e-04,
"cache_creation_input_token_cost_ultrafast": 6.25e-05,
"cache_read_input_token_cost_ultrafast": 5e-06,
}
prompt_cost, completion_cost = generic_cost_per_token(
model="gpt-5.6-sol",
usage=usage,
custom_llm_provider="openai",
service_tier="ultrafast",
model_info=model_info,
)
expected_prompt_cost = (
text_tokens * 5e-05 + cached_tokens * 5e-06 + cache_write_tokens * 6.25e-05
)
assert prompt_cost == pytest.approx(expected_prompt_cost)
assert completion_cost == pytest.approx(400 * 3e-04)
def test_service_tier_ultrafast_fallback_pricing():
"""Without *_ultrafast keys an ultrafast request bills the standard rate, not zero.
Guards the suffix fallback in _get_cost_per_unit: "_fast" is a substring of
"_ultrafast", so a shortest-first suffix match would strip the wrong suffix
and price the request at 0.
"""
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
litellm.model_cost = litellm.get_model_cost_map(url="")
usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
std_prompt_cost, std_completion_cost = generic_cost_per_token(
model="gpt-5.6-sol",
usage=usage,
custom_llm_provider="openai",
service_tier=None,
)
ultrafast_prompt_cost, ultrafast_completion_cost = generic_cost_per_token(
model="gpt-5.6-sol",
usage=usage,
custom_llm_provider="openai",
service_tier="ultrafast",
)
assert std_prompt_cost + std_completion_cost > 0
assert ultrafast_prompt_cost == pytest.approx(std_prompt_cost)
assert ultrafast_completion_cost == pytest.approx(std_completion_cost)
@pytest.mark.parametrize(
"model",
[
@ -2322,7 +2397,11 @@ def test_service_tier_suffixes_constant_in_sync_with_enum():
from litellm.litellm_core_utils.llm_cost_calc.utils import _SERVICE_TIER_SUFFIXES
from litellm.types.utils import ServiceTier
assert _SERVICE_TIER_SUFFIXES == tuple(f"_{st.value}" for st in ServiceTier)
assert set(_SERVICE_TIER_SUFFIXES) == {f"_{st.value}" for st in ServiceTier}
# longest-first so a substring match resolves "_ultrafast" before "_fast"
assert list(_SERVICE_TIER_SUFFIXES) == sorted(
_SERVICE_TIER_SUFFIXES, key=len, reverse=True
)
def test_get_cost_per_unit_falls_back_from_service_tier_key_to_base():