diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 57cc742d5c4..3203c35b64d 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -179,6 +179,8 @@ def cost_description(key: str) -> Optional[str]: return "Flex service-tier rate for the same-named base field." if key.endswith("_priority"): return "Priority service-tier rate for the same-named base field." + if "_above_" in key and key.endswith("_batches"): + return "Batch API rate applied once the prompt exceeds the token threshold in the field name." if "_above_" in key: return "Rate applied once the prompt exceeds the token threshold in the field name." return None diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index cb7da32f857..e76947f5295 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import from litellm.litellm_core_utils.llm_cost_calc.utils import ( CostCalculatorUtils, _generic_cost_per_character, + _get_batch_cost_rates, _get_regional_uplift_multiplier, _get_service_tier_cost_key, calculate_cost_component, @@ -2241,9 +2242,10 @@ def batch_cost_calculator( if not model_info: return 0.0, 0.0 - input_cost_per_token_batches: Final = model_info.get("input_cost_per_token_batches") + input_cost_per_token_batches, output_cost_per_token_batches = _get_batch_cost_rates( + model_info, usage, custom_llm_provider + ) input_cost_per_token: Final = model_info.get("input_cost_per_token") - output_cost_per_token_batches: Final = model_info.get("output_cost_per_token_batches") output_cost_per_token: Final = model_info.get("output_cost_per_token") total_prompt_cost = 0.0 total_completion_cost = 0.0 diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 8e24302b440..acb0251b021 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -61,6 +61,8 @@ _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType( ) _INCLUSIVE_THRESHOLD_PROVIDERS: Final = frozenset({"xai"}) +_BATCH_KEY_SUFFIX: Final = "_batches" +_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX) def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool: @@ -241,6 +243,40 @@ def _parse_above_token_threshold(key: str) -> float: return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1) +def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool: + return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold) + + +def _batch_rate(model_info: ModelInfo, key: str) -> float | None: + value: Final = model_info.get(key) + return value if isinstance(value, (int, float)) else None + + +def _get_batch_cost_rates( + model_info: ModelInfo, usage: Usage, custom_llm_provider: str | None +) -> tuple[float | None, float | None]: + inclusive: Final = _uses_inclusive_token_thresholds(custom_llm_provider) + tier_input_keys: Final = tuple( + key + for key, value in model_info.items() + if key.startswith("input_cost_per_token_above_") and key.endswith(_BATCH_KEY_SUFFIX) and value is not None + ) + crossed_input_key: Final = next( + ( + key + for key in sorted(tier_input_keys, key=_parse_above_token_threshold, reverse=True) + if _prompt_exceeds_threshold(usage.prompt_tokens, _parse_above_token_threshold(key), inclusive) + ), + None, + ) + flat_output_rate: Final = _batch_rate(model_info, "output_cost_per_token_batches") + if crossed_input_key is None: + return _batch_rate(model_info, "input_cost_per_token_batches"), flat_output_rate + tier_input_rate: Final = _batch_rate(model_info, crossed_input_key) + tier_output_rate: Final = _batch_rate(model_info, crossed_input_key.replace("input_", "output_", 1)) + return tier_input_rate, flat_output_rate if tier_output_rate is None else tier_output_rate + + def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict | None: tiered_pricing: Final = model_info.get("tiered_pricing") if not isinstance(tiered_pricing, list) or not tiered_pricing: @@ -559,7 +595,9 @@ def _get_token_base_cost( # so that the threshold detection loop only processes standard keys. The # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key. threshold_keys: Final = [ - k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES) + k + for k in model_info + if k.startswith("input_cost_per_token_above_") and not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) ] if not threshold_keys: return _apply_off_peak_to_base_costs( @@ -583,7 +621,7 @@ def _get_token_base_cost( # Handle both formats: _above_128k_tokens and _above_128_tokens threshold_str = key.split("_above_")[1].split("_tokens")[0] threshold = _parse_above_token_threshold(key) - if usage.prompt_tokens > threshold or (threshold_is_inclusive and usage.prompt_tokens == threshold): + if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive): # Prefer a service_tier-specific above-threshold key when available, # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini # ON_DEMAND_PRIORITY. Falls back to the standard key automatically diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7f5038e3073..cfe6abd300a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -30065,6 +30065,7 @@ "input_cost_per_token_above_272k_tokens_flex": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 4e-05, "input_cost_per_token_batches": 5e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-05, "input_cost_per_token_flex": 5e-06, "input_cost_per_token_priority": 2e-05, "litellm_provider": "openai", @@ -30077,6 +30078,7 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00015, "output_cost_per_token_batches": 2.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-05, "output_cost_per_token_flex": 2.5e-05, "output_cost_per_token_priority": 0.0001, "regional_processing_uplift_multiplier_eu": 1.1, @@ -30134,6 +30136,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -30146,6 +30149,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -30201,6 +30205,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -30213,6 +30218,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -30269,6 +30275,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, "litellm_provider": "openai", @@ -30281,6 +30288,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-06, "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, + "output_cost_per_token_above_272k_tokens_batches": 9e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -30336,6 +30344,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-07, "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_above_272k_tokens_batches": 2e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, "litellm_provider": "openai", @@ -30348,6 +30357,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-07, "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_above_272k_tokens_batches": 9e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, "regional_processing_uplift_multiplier_eu": 1.1, @@ -30619,6 +30629,7 @@ "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -30629,6 +30640,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -30676,6 +30688,7 @@ "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -30686,6 +30699,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -30833,6 +30847,7 @@ "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -30843,6 +30858,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -30885,6 +30901,7 @@ "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -30895,6 +30912,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -30935,6 +30953,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -30944,6 +30963,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -30986,6 +31006,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -30995,6 +31016,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 44c312e2e77..2539e73311f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -278,7 +278,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_video_per_second: float | None # only for vertex ai models input_cost_per_second: float | None # for OpenAI Speech models input_cost_per_token_batches: float | None + input_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token_batches: float | None + output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token: Required[float | None] output_cost_per_token_flex: float | None # OpenAI flex service tier pricing output_cost_per_token_priority: float | None # OpenAI priority service tier pricing diff --git a/litellm/utils.py b/litellm/utils.py index 8b1b32ea328..8913170bbe6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5845,7 +5845,13 @@ def _get_model_info_helper( input_cost_per_audio_per_second=_model_info.get("input_cost_per_audio_per_second", None), input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None), input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"), + input_cost_per_token_above_272k_tokens_batches=_model_info.get( + "input_cost_per_token_above_272k_tokens_batches" + ), output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"), + output_cost_per_token_above_272k_tokens_batches=_model_info.get( + "output_cost_per_token_above_272k_tokens_batches" + ), output_cost_per_token=_output_cost_per_token, output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None), output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7f5038e3073..cfe6abd300a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -30065,6 +30065,7 @@ "input_cost_per_token_above_272k_tokens_flex": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 4e-05, "input_cost_per_token_batches": 5e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-05, "input_cost_per_token_flex": 5e-06, "input_cost_per_token_priority": 2e-05, "litellm_provider": "openai", @@ -30077,6 +30078,7 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00015, "output_cost_per_token_batches": 2.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-05, "output_cost_per_token_flex": 2.5e-05, "output_cost_per_token_priority": 0.0001, "regional_processing_uplift_multiplier_eu": 1.1, @@ -30134,6 +30136,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -30146,6 +30149,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -30201,6 +30205,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -30213,6 +30218,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -30269,6 +30275,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, "litellm_provider": "openai", @@ -30281,6 +30288,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-06, "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, + "output_cost_per_token_above_272k_tokens_batches": 9e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -30336,6 +30344,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-07, "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_above_272k_tokens_batches": 2e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, "litellm_provider": "openai", @@ -30348,6 +30357,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-07, "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_above_272k_tokens_batches": 9e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, "regional_processing_uplift_multiplier_eu": 1.1, @@ -30619,6 +30629,7 @@ "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -30629,6 +30640,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -30676,6 +30688,7 @@ "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -30686,6 +30699,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -30833,6 +30847,7 @@ "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -30843,6 +30858,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -30885,6 +30901,7 @@ "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -30895,6 +30912,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -30935,6 +30953,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -30944,6 +30963,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -30986,6 +31006,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -30995,6 +31016,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index a51149bf958..1970ddef6e0 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -322,6 +322,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "input_cost_per_token_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Batch API rate applied once the prompt exceeds the token threshold in the field name." + }, "input_cost_per_token_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -508,6 +513,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "output_cost_per_token_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Batch API rate applied once the prompt exceeds the token threshold in the field name." + }, "output_cost_per_token_above_272k_tokens_flex": { "type": "number", "minimum": 0, diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index c86c7c4df03..4be668fb485 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -25,7 +25,7 @@ import respx import litellm import litellm.batches.batch_utils as bu -from litellm.types.utils import Usage +from litellm.types.utils import ModelInfo, Usage # --------------------------------------------------------------------------- # # Builders for batch OUTPUT file rows. @@ -434,6 +434,33 @@ def test_total_usage_and_cost_normalize_mixed_responses_and_chat(): assert result.cost == pytest.approx((30 * 0.00125) + (12 * 0.005)) +def test_total_cost_applies_the_long_context_batch_tier_per_line(): + long_row = _success_row(usage=_usage(300_000, 10)) + short_row = _success_row(usage=_usage(100, 10)) + + result = bu._aggregate_batch_cost_usage_models( + entries=[long_row, short_row], + custom_llm_provider="openai", + model_info=ModelInfo( + key="lit-batch-tier", + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=2e-6, + output_cost_per_token=8e-6, + litellm_provider="openai", + mode="chat", + supported_openai_params=None, + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_above_272k_tokens_batches=6e-6, + ), + ) + + assert result.cost == pytest.approx((300_000 * 2e-6) + (10 * 6e-6) + (100 * 1e-6) + (10 * 4e-6)) + + def test_total_usage_empty_is_zero(): result = bu._aggregate_batch_cost_usage_models(entries=[], custom_llm_provider="openai") assert result.cost == 0.0 diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 2046695f151..ffdc72d7383 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4492,3 +4492,57 @@ def test_batch_cost_calculator_gpt_6_astra_bills_half_the_standard_rate(_local_m assert prompt_cost == pytest.approx(1000 * 5e-6) assert completion_cost == pytest.approx(500 * 2.5e-5) + + +def test_batch_cost_calculator_bills_the_long_context_batch_tier_above_272k(_local_model_cost_map): + from litellm.cost_calculator import batch_cost_calculator + + usage = Usage(prompt_tokens=300_035, completion_tokens=64, total_tokens=300_099) + + prompt_cost, completion_cost = batch_cost_calculator(usage=usage, model="gpt-5.4", custom_llm_provider="openai") + + assert prompt_cost == pytest.approx(300_035 * 2.5e-6) + assert completion_cost == pytest.approx(64 * 1.125e-5) + + +@pytest.mark.parametrize("prompt_tokens", [272_000, 1_000]) +def test_batch_cost_calculator_bills_the_flat_batch_rate_at_or_below_272k(_local_model_cost_map, prompt_tokens): + from litellm.cost_calculator import batch_cost_calculator + + usage = Usage(prompt_tokens=prompt_tokens, completion_tokens=64, total_tokens=prompt_tokens + 64) + + prompt_cost, completion_cost = batch_cost_calculator(usage=usage, model="gpt-5.4", custom_llm_provider="openai") + + assert prompt_cost == pytest.approx(prompt_tokens * 1.25e-6) + assert completion_cost == pytest.approx(64 * 7.5e-6) + + +def test_get_model_info_exposes_the_long_context_batch_tier(_local_model_cost_map): + info = litellm.get_model_info("gpt-5.4", custom_llm_provider="openai") + + assert info["input_cost_per_token_above_272k_tokens_batches"] == 2.5e-6 + assert info["output_cost_per_token_above_272k_tokens_batches"] == 1.125e-5 + + +def test_regular_path_never_bills_the_batch_tier_keys(_local_model_cost_map, monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "lit-batch-tier-guard", + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "input_cost_per_token_batches": 1e-6, + "output_cost_per_token_batches": 4e-6, + "input_cost_per_token_above_272k_tokens_batches": 5e-6, + "output_cost_per_token_above_272k_tokens_batches": 9e-6, + }, + ) + + prompt_cost, completion_cost = litellm.cost_per_token( + model="lit-batch-tier-guard", custom_llm_provider="openai", prompt_tokens=300_035, completion_tokens=64 + ) + + assert prompt_cost == pytest.approx(300_035 * 2e-6) + assert completion_cost == pytest.approx(64 * 8e-6)