mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
fix(cost): bill batch prompts above 272K at OpenAI's long-context batch tier
This commit is contained in:
mateo-berri
parent
59d42d36e6
commit
b7f7d5c42f
10 changed files with 190 additions and 5 deletions
|
|
@ -179,6 +179,8 @@ def cost_description(key: str) -> Optional[str]:
|
|||
return "Flex service-tier rate for the same-named base field."
|
||||
if key.endswith("_priority"):
|
||||
return "Priority service-tier rate for the same-named base field."
|
||||
if "_above_" in key and key.endswith("_batches"):
|
||||
return "Batch API rate applied once the prompt exceeds the token threshold in the field name."
|
||||
if "_above_" in key:
|
||||
return "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import
|
|||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
CostCalculatorUtils,
|
||||
_generic_cost_per_character,
|
||||
_get_batch_cost_rates,
|
||||
_get_regional_uplift_multiplier,
|
||||
_get_service_tier_cost_key,
|
||||
calculate_cost_component,
|
||||
|
|
@ -2241,9 +2242,10 @@ def batch_cost_calculator(
|
|||
if not model_info:
|
||||
return 0.0, 0.0
|
||||
|
||||
input_cost_per_token_batches: Final = model_info.get("input_cost_per_token_batches")
|
||||
input_cost_per_token_batches, output_cost_per_token_batches = _get_batch_cost_rates(
|
||||
model_info, usage, custom_llm_provider
|
||||
)
|
||||
input_cost_per_token: Final = model_info.get("input_cost_per_token")
|
||||
output_cost_per_token_batches: Final = model_info.get("output_cost_per_token_batches")
|
||||
output_cost_per_token: Final = model_info.get("output_cost_per_token")
|
||||
total_prompt_cost = 0.0
|
||||
total_completion_cost = 0.0
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
|
|||
)
|
||||
|
||||
_INCLUSIVE_THRESHOLD_PROVIDERS: Final = frozenset({"xai"})
|
||||
_BATCH_KEY_SUFFIX: Final = "_batches"
|
||||
_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX)
|
||||
|
||||
|
||||
def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool:
|
||||
|
|
@ -241,6 +243,40 @@ def _parse_above_token_threshold(key: str) -> float:
|
|||
return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1)
|
||||
|
||||
|
||||
def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool:
|
||||
return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold)
|
||||
|
||||
|
||||
def _batch_rate(model_info: ModelInfo, key: str) -> float | None:
|
||||
value: Final = model_info.get(key)
|
||||
return value if isinstance(value, (int, float)) else None
|
||||
|
||||
|
||||
def _get_batch_cost_rates(
|
||||
model_info: ModelInfo, usage: Usage, custom_llm_provider: str | None
|
||||
) -> tuple[float | None, float | None]:
|
||||
inclusive: Final = _uses_inclusive_token_thresholds(custom_llm_provider)
|
||||
tier_input_keys: Final = tuple(
|
||||
key
|
||||
for key, value in model_info.items()
|
||||
if key.startswith("input_cost_per_token_above_") and key.endswith(_BATCH_KEY_SUFFIX) and value is not None
|
||||
)
|
||||
crossed_input_key: Final = next(
|
||||
(
|
||||
key
|
||||
for key in sorted(tier_input_keys, key=_parse_above_token_threshold, reverse=True)
|
||||
if _prompt_exceeds_threshold(usage.prompt_tokens, _parse_above_token_threshold(key), inclusive)
|
||||
),
|
||||
None,
|
||||
)
|
||||
flat_output_rate: Final = _batch_rate(model_info, "output_cost_per_token_batches")
|
||||
if crossed_input_key is None:
|
||||
return _batch_rate(model_info, "input_cost_per_token_batches"), flat_output_rate
|
||||
tier_input_rate: Final = _batch_rate(model_info, crossed_input_key)
|
||||
tier_output_rate: Final = _batch_rate(model_info, crossed_input_key.replace("input_", "output_", 1))
|
||||
return tier_input_rate, flat_output_rate if tier_output_rate is None else tier_output_rate
|
||||
|
||||
|
||||
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict | None:
|
||||
tiered_pricing: Final = model_info.get("tiered_pricing")
|
||||
if not isinstance(tiered_pricing, list) or not tiered_pricing:
|
||||
|
|
@ -559,7 +595,9 @@ def _get_token_base_cost(
|
|||
# so that the threshold detection loop only processes standard keys. The
|
||||
# service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key.
|
||||
threshold_keys: Final = [
|
||||
k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES)
|
||||
k
|
||||
for k in model_info
|
||||
if k.startswith("input_cost_per_token_above_") and not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES)
|
||||
]
|
||||
if not threshold_keys:
|
||||
return _apply_off_peak_to_base_costs(
|
||||
|
|
@ -583,7 +621,7 @@ def _get_token_base_cost(
|
|||
# Handle both formats: _above_128k_tokens and _above_128_tokens
|
||||
threshold_str = key.split("_above_")[1].split("_tokens")[0]
|
||||
threshold = _parse_above_token_threshold(key)
|
||||
if usage.prompt_tokens > threshold or (threshold_is_inclusive and usage.prompt_tokens == threshold):
|
||||
if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive):
|
||||
# Prefer a service_tier-specific above-threshold key when available,
|
||||
# e.g. input_cost_per_token_priority_above_200k_tokens for Gemini
|
||||
# ON_DEMAND_PRIORITY. Falls back to the standard key automatically
|
||||
|
|
|
|||
|
|
@ -30065,6 +30065,7 @@
|
|||
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 4e-05,
|
||||
"input_cost_per_token_batches": 5e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 1e-05,
|
||||
"input_cost_per_token_flex": 5e-06,
|
||||
"input_cost_per_token_priority": 2e-05,
|
||||
"litellm_provider": "openai",
|
||||
|
|
@ -30077,6 +30078,7 @@
|
|||
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 0.00015,
|
||||
"output_cost_per_token_batches": 2.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 3.75e-05,
|
||||
"output_cost_per_token_flex": 2.5e-05,
|
||||
"output_cost_per_token_priority": 0.0001,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
|
|
@ -30134,6 +30136,7 @@
|
|||
"input_cost_per_token_above_272k_tokens_flex": 4e-06,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 1.6e-05,
|
||||
"input_cost_per_token_batches": 2e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 4e-06,
|
||||
"input_cost_per_token_flex": 2e-06,
|
||||
"input_cost_per_token_priority": 8e-06,
|
||||
"litellm_provider": "openai",
|
||||
|
|
@ -30146,6 +30149,7 @@
|
|||
"output_cost_per_token_above_272k_tokens_flex": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 6e-05,
|
||||
"output_cost_per_token_batches": 1e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 1.5e-05,
|
||||
"output_cost_per_token_flex": 1e-05,
|
||||
"output_cost_per_token_priority": 4e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
|
|
@ -30201,6 +30205,7 @@
|
|||
"input_cost_per_token_above_272k_tokens_flex": 4e-06,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 1.6e-05,
|
||||
"input_cost_per_token_batches": 2e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 4e-06,
|
||||
"input_cost_per_token_flex": 2e-06,
|
||||
"input_cost_per_token_priority": 8e-06,
|
||||
"litellm_provider": "openai",
|
||||
|
|
@ -30213,6 +30218,7 @@
|
|||
"output_cost_per_token_above_272k_tokens_flex": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 6e-05,
|
||||
"output_cost_per_token_batches": 1e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 1.5e-05,
|
||||
"output_cost_per_token_flex": 1e-05,
|
||||
"output_cost_per_token_priority": 4e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
|
|
@ -30269,6 +30275,7 @@
|
|||
"input_cost_per_token_above_272k_tokens_flex": 2e-06,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 8e-06,
|
||||
"input_cost_per_token_batches": 1e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 2e-06,
|
||||
"input_cost_per_token_flex": 1e-06,
|
||||
"input_cost_per_token_priority": 4e-06,
|
||||
"litellm_provider": "openai",
|
||||
|
|
@ -30281,6 +30288,7 @@
|
|||
"output_cost_per_token_above_272k_tokens_flex": 9e-06,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.6e-05,
|
||||
"output_cost_per_token_batches": 6e-06,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 9e-06,
|
||||
"output_cost_per_token_flex": 6e-06,
|
||||
"output_cost_per_token_priority": 2.4e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
|
|
@ -30336,6 +30344,7 @@
|
|||
"input_cost_per_token_above_272k_tokens_flex": 2e-07,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
|
||||
"input_cost_per_token_batches": 1e-07,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 2e-07,
|
||||
"input_cost_per_token_flex": 1e-07,
|
||||
"input_cost_per_token_priority": 4e-07,
|
||||
"litellm_provider": "openai",
|
||||
|
|
@ -30348,6 +30357,7 @@
|
|||
"output_cost_per_token_above_272k_tokens_flex": 9e-07,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.6e-06,
|
||||
"output_cost_per_token_batches": 6e-07,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 9e-07,
|
||||
"output_cost_per_token_flex": 6e-07,
|
||||
"output_cost_per_token_priority": 2.4e-06,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
|
|
@ -30619,6 +30629,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 1e-05,
|
||||
"input_cost_per_token_flex": 2.5e-06,
|
||||
"input_cost_per_token_batches": 2.5e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 5e-06,
|
||||
"input_cost_per_token_priority": 1.25e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
|
|
@ -30629,6 +30640,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 4.5e-05,
|
||||
"output_cost_per_token_flex": 1.5e-05,
|
||||
"output_cost_per_token_batches": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 2.25e-05,
|
||||
"output_cost_per_token_priority": 7.5e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
|
|
@ -30676,6 +30688,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 1e-05,
|
||||
"input_cost_per_token_flex": 2.5e-06,
|
||||
"input_cost_per_token_batches": 2.5e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 5e-06,
|
||||
"input_cost_per_token_priority": 1.25e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
|
|
@ -30686,6 +30699,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 4.5e-05,
|
||||
"output_cost_per_token_flex": 1.5e-05,
|
||||
"output_cost_per_token_batches": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 2.25e-05,
|
||||
"output_cost_per_token_priority": 7.5e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
|
|
@ -30833,6 +30847,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 5e-06,
|
||||
"input_cost_per_token_flex": 1.25e-06,
|
||||
"input_cost_per_token_batches": 1.25e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 2.5e-06,
|
||||
"input_cost_per_token_priority": 5e-06,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
|
|
@ -30843,6 +30858,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 2.25e-05,
|
||||
"output_cost_per_token_flex": 7.5e-06,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 1.125e-05,
|
||||
"output_cost_per_token_priority": 3e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
|
|
@ -30885,6 +30901,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 5e-06,
|
||||
"input_cost_per_token_flex": 1.25e-06,
|
||||
"input_cost_per_token_batches": 1.25e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 2.5e-06,
|
||||
"input_cost_per_token_priority": 5e-06,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
|
|
@ -30895,6 +30912,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 2.25e-05,
|
||||
"output_cost_per_token_flex": 7.5e-06,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 1.125e-05,
|
||||
"output_cost_per_token_priority": 3e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
|
|
@ -30935,6 +30953,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 6e-05,
|
||||
"input_cost_per_token_flex": 1.5e-05,
|
||||
"input_cost_per_token_batches": 1.5e-05,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 3e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -30944,6 +30963,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 0.00027,
|
||||
"output_cost_per_token_flex": 9e-05,
|
||||
"output_cost_per_token_batches": 9e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 0.000135,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
"search_context_cost_per_query": {
|
||||
|
|
@ -30986,6 +31006,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 6e-05,
|
||||
"input_cost_per_token_flex": 1.5e-05,
|
||||
"input_cost_per_token_batches": 1.5e-05,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 3e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -30995,6 +31016,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 0.00027,
|
||||
"output_cost_per_token_flex": 9e-05,
|
||||
"output_cost_per_token_batches": 9e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 0.000135,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
"search_context_cost_per_query": {
|
||||
|
|
|
|||
|
|
@ -278,7 +278,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
input_cost_per_video_per_second: float | None # only for vertex ai models
|
||||
input_cost_per_second: float | None # for OpenAI Speech models
|
||||
input_cost_per_token_batches: float | None
|
||||
input_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None]
|
||||
output_cost_per_token_batches: float | None
|
||||
output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None]
|
||||
output_cost_per_token: Required[float | None]
|
||||
output_cost_per_token_flex: float | None # OpenAI flex service tier pricing
|
||||
output_cost_per_token_priority: float | None # OpenAI priority service tier pricing
|
||||
|
|
|
|||
|
|
@ -5845,7 +5845,13 @@ def _get_model_info_helper(
|
|||
input_cost_per_audio_per_second=_model_info.get("input_cost_per_audio_per_second", None),
|
||||
input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None),
|
||||
input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"),
|
||||
input_cost_per_token_above_272k_tokens_batches=_model_info.get(
|
||||
"input_cost_per_token_above_272k_tokens_batches"
|
||||
),
|
||||
output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"),
|
||||
output_cost_per_token_above_272k_tokens_batches=_model_info.get(
|
||||
"output_cost_per_token_above_272k_tokens_batches"
|
||||
),
|
||||
output_cost_per_token=_output_cost_per_token,
|
||||
output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None),
|
||||
output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None),
|
||||
|
|
|
|||
|
|
@ -30065,6 +30065,7 @@
|
|||
"input_cost_per_token_above_272k_tokens_flex": 1e-05,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 4e-05,
|
||||
"input_cost_per_token_batches": 5e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 1e-05,
|
||||
"input_cost_per_token_flex": 5e-06,
|
||||
"input_cost_per_token_priority": 2e-05,
|
||||
"litellm_provider": "openai",
|
||||
|
|
@ -30077,6 +30078,7 @@
|
|||
"output_cost_per_token_above_272k_tokens_flex": 3.75e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 0.00015,
|
||||
"output_cost_per_token_batches": 2.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 3.75e-05,
|
||||
"output_cost_per_token_flex": 2.5e-05,
|
||||
"output_cost_per_token_priority": 0.0001,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
|
|
@ -30134,6 +30136,7 @@
|
|||
"input_cost_per_token_above_272k_tokens_flex": 4e-06,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 1.6e-05,
|
||||
"input_cost_per_token_batches": 2e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 4e-06,
|
||||
"input_cost_per_token_flex": 2e-06,
|
||||
"input_cost_per_token_priority": 8e-06,
|
||||
"litellm_provider": "openai",
|
||||
|
|
@ -30146,6 +30149,7 @@
|
|||
"output_cost_per_token_above_272k_tokens_flex": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 6e-05,
|
||||
"output_cost_per_token_batches": 1e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 1.5e-05,
|
||||
"output_cost_per_token_flex": 1e-05,
|
||||
"output_cost_per_token_priority": 4e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
|
|
@ -30201,6 +30205,7 @@
|
|||
"input_cost_per_token_above_272k_tokens_flex": 4e-06,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 1.6e-05,
|
||||
"input_cost_per_token_batches": 2e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 4e-06,
|
||||
"input_cost_per_token_flex": 2e-06,
|
||||
"input_cost_per_token_priority": 8e-06,
|
||||
"litellm_provider": "openai",
|
||||
|
|
@ -30213,6 +30218,7 @@
|
|||
"output_cost_per_token_above_272k_tokens_flex": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 6e-05,
|
||||
"output_cost_per_token_batches": 1e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 1.5e-05,
|
||||
"output_cost_per_token_flex": 1e-05,
|
||||
"output_cost_per_token_priority": 4e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
|
|
@ -30269,6 +30275,7 @@
|
|||
"input_cost_per_token_above_272k_tokens_flex": 2e-06,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 8e-06,
|
||||
"input_cost_per_token_batches": 1e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 2e-06,
|
||||
"input_cost_per_token_flex": 1e-06,
|
||||
"input_cost_per_token_priority": 4e-06,
|
||||
"litellm_provider": "openai",
|
||||
|
|
@ -30281,6 +30288,7 @@
|
|||
"output_cost_per_token_above_272k_tokens_flex": 9e-06,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.6e-05,
|
||||
"output_cost_per_token_batches": 6e-06,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 9e-06,
|
||||
"output_cost_per_token_flex": 6e-06,
|
||||
"output_cost_per_token_priority": 2.4e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
|
|
@ -30336,6 +30344,7 @@
|
|||
"input_cost_per_token_above_272k_tokens_flex": 2e-07,
|
||||
"input_cost_per_token_above_272k_tokens_priority": 8e-07,
|
||||
"input_cost_per_token_batches": 1e-07,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 2e-07,
|
||||
"input_cost_per_token_flex": 1e-07,
|
||||
"input_cost_per_token_priority": 4e-07,
|
||||
"litellm_provider": "openai",
|
||||
|
|
@ -30348,6 +30357,7 @@
|
|||
"output_cost_per_token_above_272k_tokens_flex": 9e-07,
|
||||
"output_cost_per_token_above_272k_tokens_priority": 3.6e-06,
|
||||
"output_cost_per_token_batches": 6e-07,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 9e-07,
|
||||
"output_cost_per_token_flex": 6e-07,
|
||||
"output_cost_per_token_priority": 2.4e-06,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
|
|
@ -30619,6 +30629,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 1e-05,
|
||||
"input_cost_per_token_flex": 2.5e-06,
|
||||
"input_cost_per_token_batches": 2.5e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 5e-06,
|
||||
"input_cost_per_token_priority": 1.25e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
|
|
@ -30629,6 +30640,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 4.5e-05,
|
||||
"output_cost_per_token_flex": 1.5e-05,
|
||||
"output_cost_per_token_batches": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 2.25e-05,
|
||||
"output_cost_per_token_priority": 7.5e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
|
|
@ -30676,6 +30688,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 1e-05,
|
||||
"input_cost_per_token_flex": 2.5e-06,
|
||||
"input_cost_per_token_batches": 2.5e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 5e-06,
|
||||
"input_cost_per_token_priority": 1.25e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
|
|
@ -30686,6 +30699,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 4.5e-05,
|
||||
"output_cost_per_token_flex": 1.5e-05,
|
||||
"output_cost_per_token_batches": 1.5e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 2.25e-05,
|
||||
"output_cost_per_token_priority": 7.5e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
|
|
@ -30833,6 +30847,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 5e-06,
|
||||
"input_cost_per_token_flex": 1.25e-06,
|
||||
"input_cost_per_token_batches": 1.25e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 2.5e-06,
|
||||
"input_cost_per_token_priority": 5e-06,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
|
|
@ -30843,6 +30858,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 2.25e-05,
|
||||
"output_cost_per_token_flex": 7.5e-06,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 1.125e-05,
|
||||
"output_cost_per_token_priority": 3e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
|
|
@ -30885,6 +30901,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 5e-06,
|
||||
"input_cost_per_token_flex": 1.25e-06,
|
||||
"input_cost_per_token_batches": 1.25e-06,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 2.5e-06,
|
||||
"input_cost_per_token_priority": 5e-06,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
|
|
@ -30895,6 +30912,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 2.25e-05,
|
||||
"output_cost_per_token_flex": 7.5e-06,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 1.125e-05,
|
||||
"output_cost_per_token_priority": 3e-05,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
|
|
@ -30935,6 +30953,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 6e-05,
|
||||
"input_cost_per_token_flex": 1.5e-05,
|
||||
"input_cost_per_token_batches": 1.5e-05,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 3e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -30944,6 +30963,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 0.00027,
|
||||
"output_cost_per_token_flex": 9e-05,
|
||||
"output_cost_per_token_batches": 9e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 0.000135,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
"search_context_cost_per_query": {
|
||||
|
|
@ -30986,6 +31006,7 @@
|
|||
"input_cost_per_token_above_272k_tokens": 6e-05,
|
||||
"input_cost_per_token_flex": 1.5e-05,
|
||||
"input_cost_per_token_batches": 1.5e-05,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 3e-05,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 1050000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -30995,6 +31016,7 @@
|
|||
"output_cost_per_token_above_272k_tokens": 0.00027,
|
||||
"output_cost_per_token_flex": 9e-05,
|
||||
"output_cost_per_token_batches": 9e-05,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 0.000135,
|
||||
"regional_processing_uplift_multiplier_eu": 1.1,
|
||||
"regional_processing_uplift_multiplier_us": 1.1,
|
||||
"search_context_cost_per_query": {
|
||||
|
|
|
|||
|
|
@ -322,6 +322,11 @@
|
|||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_token_above_272k_tokens_batches": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Batch API rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_token_above_272k_tokens_flex": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
|
|
@ -508,6 +513,11 @@
|
|||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_token_above_272k_tokens_batches": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"description": "Batch API rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_token_above_272k_tokens_flex": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import respx
|
|||
|
||||
import litellm
|
||||
import litellm.batches.batch_utils as bu
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Builders for batch OUTPUT file rows.
|
||||
|
|
@ -434,6 +434,33 @@ def test_total_usage_and_cost_normalize_mixed_responses_and_chat():
|
|||
assert result.cost == pytest.approx((30 * 0.00125) + (12 * 0.005))
|
||||
|
||||
|
||||
def test_total_cost_applies_the_long_context_batch_tier_per_line():
|
||||
long_row = _success_row(usage=_usage(300_000, 10))
|
||||
short_row = _success_row(usage=_usage(100, 10))
|
||||
|
||||
result = bu._aggregate_batch_cost_usage_models(
|
||||
entries=[long_row, short_row],
|
||||
custom_llm_provider="openai",
|
||||
model_info=ModelInfo(
|
||||
key="lit-batch-tier",
|
||||
max_tokens=None,
|
||||
max_input_tokens=None,
|
||||
max_output_tokens=None,
|
||||
input_cost_per_token=2e-6,
|
||||
output_cost_per_token=8e-6,
|
||||
litellm_provider="openai",
|
||||
mode="chat",
|
||||
supported_openai_params=None,
|
||||
input_cost_per_token_batches=1e-6,
|
||||
output_cost_per_token_batches=4e-6,
|
||||
input_cost_per_token_above_272k_tokens_batches=2e-6,
|
||||
output_cost_per_token_above_272k_tokens_batches=6e-6,
|
||||
),
|
||||
)
|
||||
|
||||
assert result.cost == pytest.approx((300_000 * 2e-6) + (10 * 6e-6) + (100 * 1e-6) + (10 * 4e-6))
|
||||
|
||||
|
||||
def test_total_usage_empty_is_zero():
|
||||
result = bu._aggregate_batch_cost_usage_models(entries=[], custom_llm_provider="openai")
|
||||
assert result.cost == 0.0
|
||||
|
|
|
|||
|
|
@ -4492,3 +4492,57 @@ def test_batch_cost_calculator_gpt_6_astra_bills_half_the_standard_rate(_local_m
|
|||
|
||||
assert prompt_cost == pytest.approx(1000 * 5e-6)
|
||||
assert completion_cost == pytest.approx(500 * 2.5e-5)
|
||||
|
||||
|
||||
def test_batch_cost_calculator_bills_the_long_context_batch_tier_above_272k(_local_model_cost_map):
|
||||
from litellm.cost_calculator import batch_cost_calculator
|
||||
|
||||
usage = Usage(prompt_tokens=300_035, completion_tokens=64, total_tokens=300_099)
|
||||
|
||||
prompt_cost, completion_cost = batch_cost_calculator(usage=usage, model="gpt-5.4", custom_llm_provider="openai")
|
||||
|
||||
assert prompt_cost == pytest.approx(300_035 * 2.5e-6)
|
||||
assert completion_cost == pytest.approx(64 * 1.125e-5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt_tokens", [272_000, 1_000])
|
||||
def test_batch_cost_calculator_bills_the_flat_batch_rate_at_or_below_272k(_local_model_cost_map, prompt_tokens):
|
||||
from litellm.cost_calculator import batch_cost_calculator
|
||||
|
||||
usage = Usage(prompt_tokens=prompt_tokens, completion_tokens=64, total_tokens=prompt_tokens + 64)
|
||||
|
||||
prompt_cost, completion_cost = batch_cost_calculator(usage=usage, model="gpt-5.4", custom_llm_provider="openai")
|
||||
|
||||
assert prompt_cost == pytest.approx(prompt_tokens * 1.25e-6)
|
||||
assert completion_cost == pytest.approx(64 * 7.5e-6)
|
||||
|
||||
|
||||
def test_get_model_info_exposes_the_long_context_batch_tier(_local_model_cost_map):
|
||||
info = litellm.get_model_info("gpt-5.4", custom_llm_provider="openai")
|
||||
|
||||
assert info["input_cost_per_token_above_272k_tokens_batches"] == 2.5e-6
|
||||
assert info["output_cost_per_token_above_272k_tokens_batches"] == 1.125e-5
|
||||
|
||||
|
||||
def test_regular_path_never_bills_the_batch_tier_keys(_local_model_cost_map, monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"lit-batch-tier-guard",
|
||||
{
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 2e-6,
|
||||
"output_cost_per_token": 8e-6,
|
||||
"input_cost_per_token_batches": 1e-6,
|
||||
"output_cost_per_token_batches": 4e-6,
|
||||
"input_cost_per_token_above_272k_tokens_batches": 5e-6,
|
||||
"output_cost_per_token_above_272k_tokens_batches": 9e-6,
|
||||
},
|
||||
)
|
||||
|
||||
prompt_cost, completion_cost = litellm.cost_per_token(
|
||||
model="lit-batch-tier-guard", custom_llm_provider="openai", prompt_tokens=300_035, completion_tokens=64
|
||||
)
|
||||
|
||||
assert prompt_cost == pytest.approx(300_035 * 2e-6)
|
||||
assert completion_cost == pytest.approx(64 * 8e-6)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue