scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-29 01:42:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 79

feat(cost): balanced service tier

Browse source
This commit is contained in:
shrey kharbanda 2026-09-24 04:57:26 +00:00
parent 6b036ddcc1
commit c208a9c9ad
4 changed files with 43 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -69,6 +69,7 @@ _SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple(
_SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
{
ServiceTier.BALANCED.value: ServiceTier.BALANCED.value,
ServiceTier.FLEX.value: ServiceTier.FLEX.value,
ServiceTier.PRIORITY.value: ServiceTier.PRIORITY.value,
ServiceTier.FAST.value: ServiceTier.PRIORITY.value,
@ -252,7 +253,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str:
Args:
base_key: The base cost key (e.g., "input_cost_per_token")
service_tier: The service tier ("flex", "priority", "fast", "ultrafast", or None for standard)
service_tier: The service tier ("balanced", "flex", "priority", "fast", "ultrafast", or None for standard)
Returns:
str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")

19
litellm/types/utils.py
View file

@ -267,6 +267,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
max_input_tokens: Required[int | None]
max_output_tokens: Required[int | None]
input_cost_per_token: Required[float | None]
input_cost_per_token_balanced: float | None # balanced service tier pricing
input_cost_per_token_flex: float | None # OpenAI flex service tier pricing
input_cost_per_token_priority: float | None # OpenAI priority service tier pricing
input_cost_per_token_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
@ -275,14 +276,17 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_creation_input_token_cost_above_272k_tokens: float | None
cache_creation_input_token_cost_above_272k_tokens_priority: float | None
cache_creation_input_token_cost_above_272k_tokens_flex: float | None
cache_creation_input_token_cost_above_272k_tokens_balanced: float | None
cache_creation_input_token_cost_above_1hr: float | None
cache_creation_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_creation_input_token_cost_balanced: float | None # balanced service tier pricing
cache_creation_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_creation_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_read_input_token_cost: float | None
cache_read_input_audio_token_cost: ReadOnly[float | None]
cache_read_input_image_token_cost: ReadOnly[float | None]
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
cache_read_input_token_cost_balanced: float | None # balanced service tier pricing
cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing
cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
cache_read_input_token_cost_above_200k_tokens: float | None
@ -290,6 +294,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
cache_read_input_token_cost_above_272k_tokens: float | None
cache_read_input_token_cost_above_272k_tokens_priority: float | None
cache_read_input_token_cost_above_272k_tokens_flex: float | None
cache_read_input_token_cost_above_272k_tokens_balanced: float | None
cache_read_input_token_cost_above_512k_tokens: float | None
cache_read_input_token_cost_batches: ReadOnly[float | None]
cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None]
@ -307,6 +312,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
input_cost_per_token_above_272k_tokens: float | None # GPT-5.4/5.4-pro: prompts >272K priced at 2x input
input_cost_per_token_above_272k_tokens_priority: float | None
input_cost_per_token_above_272k_tokens_flex: float | None
input_cost_per_token_above_272k_tokens_balanced: float | None
input_cost_per_token_above_512k_tokens: float | None # MiniMax-M3: prompts >512K priced at 2x input
input_cost_per_character_above_128k_tokens: float | None # only for vertex ai models
input_cost_per_query: float | None # per-request pricing: rerank, search, and Bedrock Marengo embeddings
@ -324,6 +330,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
output_cost_per_token_batches: float | None
output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None]
output_cost_per_token: Required[float | None]
output_cost_per_token_balanced: float | None # balanced service tier pricing
output_cost_per_token_flex: float | None # OpenAI flex service tier pricing
output_cost_per_token_priority: float | None # OpenAI priority service tier pricing
output_cost_per_token_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
@ -344,6 +351,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
output_cost_per_token_above_272k_tokens: float | None # GPT-5.4/5.4-pro: prompts >272K priced at 1.5x output
output_cost_per_token_above_272k_tokens_priority: float | None
output_cost_per_token_above_272k_tokens_flex: float | None
output_cost_per_token_above_272k_tokens_balanced: float | None
output_cost_per_token_above_512k_tokens: float | None # MiniMax-M3: prompts >512K priced at 2x output
output_cost_per_character_above_128k_tokens: float | None # only for vertex ai models
output_cost_per_image: float | None
@ -353,6 +361,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
output_vector_size: int | None
output_cost_per_reasoning_token: float | None
output_cost_per_reasoning_token_flex: float | None
output_cost_per_reasoning_token_balanced: float | None
output_cost_per_reasoning_token_priority: float | None
output_cost_per_video_per_second: float | None # only for vertex ai models
output_cost_per_audio_per_second: float | None # only for vertex ai models
@ -3706,6 +3715,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
# Include all ModelInfoBase fields as optional
# This allows any model_info parameter to be set in litellm_params
input_cost_per_token_balanced: float | None = None
input_cost_per_token_flex: float | None = None
input_cost_per_token_priority: float | None = None
input_cost_per_token_ultrafast: float | None = None
@ -3714,17 +3724,21 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
cache_creation_input_token_cost_above_272k_tokens: float | None = None
cache_creation_input_token_cost_above_272k_tokens_priority: float | None = None
cache_creation_input_token_cost_above_272k_tokens_flex: float | None = None
cache_creation_input_token_cost_above_272k_tokens_balanced: float | None = None
cache_creation_input_token_cost_flex: float | None = None
cache_creation_input_token_cost_balanced: float | None = None
cache_creation_input_token_cost_priority: float | None = None
cache_creation_input_token_cost_ultrafast: float | None = None
cache_creation_input_audio_token_cost: float | None = None
cache_read_input_token_cost_flex: float | None = None
cache_read_input_token_cost_balanced: float | None = None
cache_read_input_token_cost_priority: float | None = None
cache_read_input_token_cost_ultrafast: float | None = None
cache_read_input_token_cost_above_200k_tokens: float | None = None
cache_read_input_token_cost_above_200k_tokens_priority: float | None = None
cache_read_input_token_cost_above_272k_tokens_priority: float | None = None
cache_read_input_token_cost_above_272k_tokens_flex: float | None = None
cache_read_input_token_cost_above_272k_tokens_balanced: float | None = None
cache_read_input_token_cost_batches: float | None = None
cache_read_input_token_cost_above_272k_tokens_batches: float | None = None
cache_creation_input_token_cost_batches: float | None = None
@ -3739,6 +3753,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
input_cost_per_token_above_200k_tokens_priority: float | None = None
input_cost_per_token_above_272k_tokens_priority: float | None = None
input_cost_per_token_above_272k_tokens_flex: float | None = None
input_cost_per_token_above_272k_tokens_balanced: float | None = None
input_cost_per_token_above_272k_tokens_batches: float | None = None
input_cost_per_query: float | None = None
input_cost_per_image: float | None = None
@ -3754,6 +3769,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
input_cost_per_token_batches: float | None = None
input_cost_per_video_token_batches: float | None = None
output_cost_per_token_batches: float | None = None
output_cost_per_token_balanced: float | None = None
output_cost_per_token_flex: float | None = None
output_cost_per_token_priority: float | None = None
output_cost_per_token_ultrafast: float | None = None
@ -3763,6 +3779,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
output_cost_per_token_above_200k_tokens_priority: float | None = None
output_cost_per_token_above_272k_tokens_priority: float | None = None
output_cost_per_token_above_272k_tokens_flex: float | None = None
output_cost_per_token_above_272k_tokens_balanced: float | None = None
output_cost_per_token_above_272k_tokens_batches: float | None = None
output_cost_per_character_above_128k_tokens: float | None = None
output_cost_per_image: float | None = None
@ -3770,6 +3787,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
output_cost_per_video_token: float | None = None
output_cost_per_reasoning_token: float | None = None
output_cost_per_reasoning_token_flex: float | None = None
output_cost_per_reasoning_token_balanced: float | None = None
output_cost_per_reasoning_token_priority: float | None = None
output_cost_per_video_per_second: float | None = None
output_cost_per_audio_per_second: float | None = None
@ -4559,6 +4577,7 @@ class ServiceTier(Enum):
"""Enum for service tier types used in cost calculations."""
AUTO = "auto"
BALANCED = "balanced"
FLEX = "flex"
PRIORITY = "priority"
FAST = "fast"

21
litellm/utils.py
View file

@ -6058,6 +6058,7 @@ def _get_model_info_helper(
max_input_tokens=_model_info.get("max_input_tokens", None),
max_output_tokens=_model_info.get("max_output_tokens", None),
input_cost_per_token=_input_cost_per_token,
input_cost_per_token_balanced=_model_info.get("input_cost_per_token_balanced", None),
input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None),
input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None),
input_cost_per_token_ultrafast=_model_info.get("input_cost_per_token_ultrafast", None),
@ -6074,7 +6075,13 @@ def _get_model_info_helper(
cache_creation_input_token_cost_above_272k_tokens_flex=_model_info.get(
"cache_creation_input_token_cost_above_272k_tokens_flex", None
),
cache_creation_input_token_cost_above_272k_tokens_balanced=_model_info.get(
"cache_creation_input_token_cost_above_272k_tokens_balanced", None
),
cache_creation_input_token_cost_flex=_model_info.get("cache_creation_input_token_cost_flex", None),
cache_creation_input_token_cost_balanced=_model_info.get(
"cache_creation_input_token_cost_balanced", None
),
cache_creation_input_token_cost_priority=_model_info.get(
"cache_creation_input_token_cost_priority", None
),
@ -6099,10 +6106,14 @@ def _get_model_info_helper(
cache_read_input_token_cost_above_272k_tokens_flex=_model_info.get(
"cache_read_input_token_cost_above_272k_tokens_flex", None
),
cache_read_input_token_cost_above_272k_tokens_balanced=_model_info.get(
"cache_read_input_token_cost_above_272k_tokens_balanced", None
),
cache_read_input_token_cost_above_512k_tokens=_model_info.get(
"cache_read_input_token_cost_above_512k_tokens", None
),
cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None),
cache_read_input_token_cost_balanced=_model_info.get("cache_read_input_token_cost_balanced", None),
cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None),
cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None),
cache_read_input_token_cost_batches=_model_info.get("cache_read_input_token_cost_batches"),
@ -6130,6 +6141,9 @@ def _get_model_info_helper(
input_cost_per_token_above_272k_tokens_flex=_model_info.get(
"input_cost_per_token_above_272k_tokens_flex", None
),
input_cost_per_token_above_272k_tokens_balanced=_model_info.get(
"input_cost_per_token_above_272k_tokens_balanced", None
),
input_cost_per_token_above_512k_tokens=_model_info.get("input_cost_per_token_above_512k_tokens", None),
input_cost_per_query=_model_info.get("input_cost_per_query", None),
input_cost_per_second=_model_info.get("input_cost_per_second", None),
@ -6151,6 +6165,7 @@ def _get_model_info_helper(
"output_cost_per_token_above_272k_tokens_batches"
),
output_cost_per_token=_output_cost_per_token,
output_cost_per_token_balanced=_model_info.get("output_cost_per_token_balanced", None),
output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None),
output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None),
output_cost_per_token_ultrafast=_model_info.get("output_cost_per_token_ultrafast", None),
@ -6165,6 +6180,9 @@ def _get_model_info_helper(
output_cost_per_character=_model_info.get("output_cost_per_character", None),
output_cost_per_reasoning_token=_model_info.get("output_cost_per_reasoning_token", None),
output_cost_per_reasoning_token_flex=_model_info.get("output_cost_per_reasoning_token_flex", None),
output_cost_per_reasoning_token_balanced=_model_info.get(
"output_cost_per_reasoning_token_balanced", None
),
output_cost_per_reasoning_token_priority=_model_info.get(
"output_cost_per_reasoning_token_priority", None
),
@ -6189,6 +6207,9 @@ def _get_model_info_helper(
output_cost_per_token_above_272k_tokens_flex=_model_info.get(
"output_cost_per_token_above_272k_tokens_flex", None
),
output_cost_per_token_above_272k_tokens_balanced=_model_info.get(
"output_cost_per_token_above_272k_tokens_balanced", None
),
output_cost_per_token_above_512k_tokens=_model_info.get(
"output_cost_per_token_above_512k_tokens", None
),

2
tests/test_litellm/test_model_prices_schema.py
View file

@ -202,7 +202,7 @@ def test_price_map_has_no_duplicate_keys(path: Path):
DATED_VARIANT = re.compile(r"^(.*?)-(\d{4}-\d{2}-\d{2})$")
SERVICE_TIER_SUFFIXES = ("_flex", "_priority")
SERVICE_TIER_SUFFIXES = ("_balanced", "_flex", "_priority")
def tier_anchor(tier_key: str) -> str: