diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 9d6ad8b6e39..f73c4942a1c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -42,14 +42,18 @@ _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency) # Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per # request in the cost-calc path, so the f-strings are built once here instead -# of being rebuilt for every model_info key on every call. -_SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple(f"_{st.value}" for st in ServiceTier) +# of being rebuilt for every model_info key on every call. Longest-first so a +# substring match resolves "_ultrafast" before "_fast". +_SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple( + sorted((f"_{st.value}" for st in ServiceTier), key=len, reverse=True) +) _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType( { ServiceTier.FLEX.value: ServiceTier.FLEX.value, ServiceTier.PRIORITY.value: ServiceTier.PRIORITY.value, ServiceTier.FAST.value: ServiceTier.PRIORITY.value, + ServiceTier.ULTRAFAST.value: ServiceTier.ULTRAFAST.value, } ) @@ -191,7 +195,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str: Args: base_key: The base cost key (e.g., "input_cost_per_token") - service_tier: The service tier ("flex", "priority", "fast", or None for standard) + service_tier: The service tier ("flex", "priority", "fast", "ultrafast", or None for standard) Returns: str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token") diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 13831799c7f..418f45c291a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -196,6 +196,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_token: Required[float | None] input_cost_per_token_flex: float | None # OpenAI flex service tier pricing input_cost_per_token_priority: float | None # OpenAI priority service tier pricing + input_cost_per_token_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_creation_input_token_cost: float | None cache_creation_input_token_cost_above_200k_tokens: float | None cache_creation_input_token_cost_above_272k_tokens: float | None @@ -204,9 +205,11 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_creation_input_token_cost_above_1hr: float | None cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing + cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_read_input_token_cost: float | None cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing + cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_read_input_token_cost_above_200k_tokens: float | None cache_read_input_token_cost_above_200k_tokens_priority: float | None cache_read_input_token_cost_above_272k_tokens: float | None @@ -238,6 +241,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_token: Required[float | None] output_cost_per_token_flex: float | None # OpenAI flex service tier pricing output_cost_per_token_priority: float | None # OpenAI priority service tier pricing + output_cost_per_token_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing regional_processing_uplift_multiplier_eu: ( float | None ) # OpenAI EU data-residency uplift multiplier applied to all token costs (e.g. 1.10 = +10%) @@ -3291,6 +3295,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): # This allows any model_info parameter to be set in litellm_params input_cost_per_token_flex: float | None = None input_cost_per_token_priority: float | None = None + input_cost_per_token_ultrafast: float | None = None cache_creation_input_token_cost_above_1hr: float | None = None cache_creation_input_token_cost_above_200k_tokens: float | None = None cache_creation_input_token_cost_above_272k_tokens: float | None = None @@ -3298,9 +3303,11 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_creation_input_token_cost_above_272k_tokens_flex: float | None = None cache_creation_input_token_cost_flex: float | None = None cache_creation_input_token_cost_priority: float | None = None + cache_creation_input_token_cost_ultrafast: float | None = None cache_creation_input_audio_token_cost: float | None = None cache_read_input_token_cost_flex: float | None = None cache_read_input_token_cost_priority: float | None = None + cache_read_input_token_cost_ultrafast: float | None = None cache_read_input_token_cost_above_200k_tokens: float | None = None cache_read_input_token_cost_above_200k_tokens_priority: float | None = None cache_read_input_token_cost_above_272k_tokens_priority: float | None = None @@ -3327,6 +3334,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_token_batches: float | None = None output_cost_per_token_flex: float | None = None output_cost_per_token_priority: float | None = None + output_cost_per_token_ultrafast: float | None = None output_cost_per_audio_token: float | None = None output_cost_per_token_above_128k_tokens: float | None = None output_cost_per_token_above_200k_tokens: float | None = None @@ -3994,6 +4002,7 @@ class ServiceTier(Enum): FLEX = "flex" PRIORITY = "priority" FAST = "fast" + ULTRAFAST = "ultrafast" class DataResidency(Enum): diff --git a/litellm/utils.py b/litellm/utils.py index 68f4278c87a..316fdf00aa2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5578,6 +5578,7 @@ def _get_model_info_helper( input_cost_per_token=_input_cost_per_token, input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None), input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None), + input_cost_per_token_ultrafast=_model_info.get("input_cost_per_token_ultrafast", None), cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None), cache_creation_input_token_cost_above_200k_tokens=_model_info.get( "cache_creation_input_token_cost_above_200k_tokens", None @@ -5595,6 +5596,9 @@ def _get_model_info_helper( cache_creation_input_token_cost_priority=_model_info.get( "cache_creation_input_token_cost_priority", None ), + cache_creation_input_token_cost_ultrafast=_model_info.get( + "cache_creation_input_token_cost_ultrafast", None + ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( @@ -5617,6 +5621,7 @@ def _get_model_info_helper( ), cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None), cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None), + cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None), cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), @@ -5647,6 +5652,7 @@ def _get_model_info_helper( output_cost_per_token=_output_cost_per_token, output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None), output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None), + output_cost_per_token_ultrafast=_model_info.get("output_cost_per_token_ultrafast", None), regional_processing_uplift_multiplier_eu=_model_info.get( "regional_processing_uplift_multiplier_eu", None ), diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 1402e056b72..1826f56d667 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1781,6 +1781,81 @@ def test_service_tier_fallback_pricing(): ), f"Standard completion cost mismatch: {std_cost[1]} vs {expected_standard_completion}" +def test_service_tier_ultrafast_pricing(): + """An ultrafast request bills the *_ultrafast rates for all token types. + + Regression for the ultrafast service tier being absent from ServiceTier: + the cost-key lookup silently returned the standard keys, undercounting + every ultrafast request. + """ + cached_tokens = 200 + cache_write_tokens = 300 + text_tokens = 500 + usage = Usage( + prompt_tokens=text_tokens + cached_tokens + cache_write_tokens, + completion_tokens=400, + total_tokens=text_tokens + cached_tokens + cache_write_tokens + 400, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=cached_tokens, cache_write_tokens=cache_write_tokens + ), + ) + model_info: ModelInfo = { + "key": "gpt-5.6-sol", + "input_cost_per_token": 5e-06, + "output_cost_per_token": 3e-05, + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token_ultrafast": 5e-05, + "output_cost_per_token_ultrafast": 3e-04, + "cache_creation_input_token_cost_ultrafast": 6.25e-05, + "cache_read_input_token_cost_ultrafast": 5e-06, + } + + prompt_cost, completion_cost = generic_cost_per_token( + model="gpt-5.6-sol", + usage=usage, + custom_llm_provider="openai", + service_tier="ultrafast", + model_info=model_info, + ) + + expected_prompt_cost = ( + text_tokens * 5e-05 + cached_tokens * 5e-06 + cache_write_tokens * 6.25e-05 + ) + assert prompt_cost == pytest.approx(expected_prompt_cost) + assert completion_cost == pytest.approx(400 * 3e-04) + + +def test_service_tier_ultrafast_fallback_pricing(): + """Without *_ultrafast keys an ultrafast request bills the standard rate, not zero. + + Guards the suffix fallback in _get_cost_per_unit: "_fast" is a substring of + "_ultrafast", so a shortest-first suffix match would strip the wrong suffix + and price the request at 0. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500) + + std_prompt_cost, std_completion_cost = generic_cost_per_token( + model="gpt-5.6-sol", + usage=usage, + custom_llm_provider="openai", + service_tier=None, + ) + ultrafast_prompt_cost, ultrafast_completion_cost = generic_cost_per_token( + model="gpt-5.6-sol", + usage=usage, + custom_llm_provider="openai", + service_tier="ultrafast", + ) + + assert std_prompt_cost + std_completion_cost > 0 + assert ultrafast_prompt_cost == pytest.approx(std_prompt_cost) + assert ultrafast_completion_cost == pytest.approx(std_completion_cost) + + @pytest.mark.parametrize( "model", [ @@ -2322,7 +2397,11 @@ def test_service_tier_suffixes_constant_in_sync_with_enum(): from litellm.litellm_core_utils.llm_cost_calc.utils import _SERVICE_TIER_SUFFIXES from litellm.types.utils import ServiceTier - assert _SERVICE_TIER_SUFFIXES == tuple(f"_{st.value}" for st in ServiceTier) + assert set(_SERVICE_TIER_SUFFIXES) == {f"_{st.value}" for st in ServiceTier} + # longest-first so a substring match resolves "_ultrafast" before "_fast" + assert list(_SERVICE_TIER_SUFFIXES) == sorted( + _SERVICE_TIER_SUFFIXES, key=len, reverse=True + ) def test_get_cost_per_unit_falls_back_from_service_tier_key_to_base():