diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7b02a5ede00..ae54b91c5a0 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( _get_service_tier_cost_key, calculate_cost_component, generic_cost_per_token, + get_batch_cost_rates, get_billable_input_tokens, get_token_type_cost_breakdown, parse_prompt_tokens_details, @@ -2557,35 +2558,14 @@ def batch_cost_calculator( if not model_info: return 0.0, 0.0 - input_cost_per_token_batches: Final = model_info.get("input_cost_per_token_batches") + batch_rates: Final = get_batch_cost_rates(model_info, usage, custom_llm_provider) input_cost_per_token: Final = model_info.get("input_cost_per_token") - output_cost_per_token_batches: Final = model_info.get("output_cost_per_token_batches") output_cost_per_token: Final = model_info.get("output_cost_per_token") total_prompt_cost = 0.0 total_completion_cost = 0.0 - if input_cost_per_token_batches is not None: - batch_details: Final = parse_prompt_tokens_details(usage) - audio_tokens, image_tokens, video_tokens = ( - batch_details["audio_tokens"], - batch_details["image_tokens"], - batch_details["video_tokens"], - ) - modality_rates: Final = ( - _batch_rate(model_info, "input_cost_per_audio_token_batches", input_cost_per_token_batches), - _batch_rate(model_info, "input_cost_per_image_token_batches", input_cost_per_token_batches), - _batch_rate(model_info, "input_cost_per_video_token_batches", input_cost_per_token_batches), - ) - total_prompt_cost = sum( - tokens * rate - for tokens, rate in zip( - ( - max((usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens, 0), - audio_tokens, - image_tokens, - video_tokens, - ), - (input_cost_per_token_batches, *modality_rates), - ) + if batch_rates.input is not None: + total_prompt_cost = _batch_prompt_cost( + usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation ) elif input_cost_per_token: details: Final = parse_prompt_tokens_details(usage) @@ -2605,8 +2585,8 @@ def batch_cost_calculator( cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2 - if output_cost_per_token_batches is not None: - total_completion_cost = usage.completion_tokens * output_cost_per_token_batches + if batch_rates.output is not None: + total_completion_cost = usage.completion_tokens * batch_rates.output elif output_cost_per_token: total_completion_cost = ( usage.completion_tokens * (output_cost_per_token) / 2 @@ -2620,6 +2600,34 @@ def batch_cost_calculator( return total_prompt_cost, total_completion_cost +def _batch_prompt_cost( + usage: Usage, + model_info: ModelInfo, + input_rate: float, + cache_read_rate: float | None, + cache_creation_rate: float | None, +) -> float: + details: Final = parse_prompt_tokens_details(usage) + cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 + written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0 + audio_tokens, image_tokens, video_tokens = ( + details["audio_tokens"], + details["image_tokens"], + details["video_tokens"], + ) + text_tokens: Final = max( + (usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens - cached_tokens - written_tokens, 0 + ) + return ( + text_tokens * input_rate + + audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate) + + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate) + + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) + + cached_tokens * (cache_read_rate or 0.0) + + written_tokens * (cache_creation_rate or 0.0) + ) + + def _attribute_value(obj: object, name: str) -> object: return getattr(obj, name) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 306812cca0a..357ee51a44a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -387,11 +387,40 @@ _DEPLOYMENT_PRICING_KEYS: Final = ( "output_cost_per_token", "input_cost_per_token_batches", "output_cost_per_token_batches", + "input_cost_per_token_above_272k_tokens_batches", + "output_cost_per_token_above_272k_tokens_batches", + "cache_read_input_token_cost_batches", + "cache_read_input_token_cost_above_272k_tokens_batches", + "cache_creation_input_token_cost_batches", + "cache_creation_input_token_cost_above_272k_tokens_batches", "ocr_cost_per_page", "ocr_cost_per_page_batches", "annotation_cost_per_page", "annotation_cost_per_page_batches", ) +_INPUT_PRICING_KEY_PREFIXES: Final = ( + "input_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", +) +_OUTPUT_PRICING_KEY_PREFIXES: Final = ("output_cost_per_token",) +_BATCH_PRICING_KEY_SUFFIX: Final = "_batches" + + +_NO_CARRIED_RATES: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _published_direction( + published: ModelInfo, registered: Mapping[str, object], flat_key: str, prefixes: tuple[str, ...] +) -> Mapping[str, object]: + return MappingProxyType( + { + key: value + for key, value in published.items() + if registered.get(key) is None + and (key == flat_key or (key.startswith(prefixes) and key.endswith(_BATCH_PRICING_KEY_SUFFIX))) + } + ) def deployment_pricing_model_info(model_id: str | None, deployment_model: str | None) -> ModelInfo | None: @@ -403,12 +432,15 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str | get_model_info fills absent costs with 0, so asking it directly cannot tell "configured as free" apart from "no pricing configured". A deployment may declare only one side of its pricing, so the side it leaves out keeps - the model's published rates instead of billing as zero. Ownership is per - token direction: declaring either rate for a direction takes that whole - direction, so a published batch rate can never displace a standard rate - the deployment configured itself. OCR per-page rates count as declared - pricing too; they pass through as registered and ``ocr_batch_cost`` layers - the published rate under each per-page family the deployment leaves out. + the model's published standard rate and every published batch rate for + that direction (flat, long-context tier, cached, cache write) instead of + billing as zero. Ownership is per token direction: declaring the flat + standard or flat batch rate for a direction takes that whole direction, so + a published batch rate can never displace a standard rate the deployment + configured itself. A tier-only override keeps every published rate it left + out. OCR per-page rates count as declared pricing too; they pass through as + registered and ``ocr_batch_cost`` layers the published rate under each + per-page family the deployment leaves out. """ if model_id is None: return None @@ -429,13 +461,22 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str | registered.get("output_cost_per_token") is not None or registered.get("output_cost_per_token_batches") is not None ) - if not declares_input: - merged["input_cost_per_token"] = published.get("input_cost_per_token") - merged["input_cost_per_token_batches"] = published.get("input_cost_per_token_batches") - if not declares_output: - merged["output_cost_per_token"] = published.get("output_cost_per_token") - merged["output_cost_per_token_batches"] = published.get("output_cost_per_token_batches") - return merged + carried_input: Final = ( + _NO_CARRIED_RATES + if declares_input + else _published_direction(published, registered, "input_cost_per_token", _INPUT_PRICING_KEY_PREFIXES) + ) + carried_output: Final = ( + _NO_CARRIED_RATES + if declares_output + else _published_direction(published, registered, "output_cost_per_token", _OUTPUT_PRICING_KEY_PREFIXES) + ) + priced: Final[ModelInfo] = { # pyright: ignore[reportAssignmentType] # carried keys are ModelInfo rates + **merged, + **carried_input, + **carried_output, + } + return priced def _published_pricing(deployment_model: str | None) -> ModelInfo | None: diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index c60e3089816..3845f50af79 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -67,6 +67,15 @@ _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType( ) _INCLUSIVE_THRESHOLD_PROVIDERS: Final = frozenset({"xai"}) +_BATCH_KEY_SUFFIX: Final = "_batches" +_BATCH_RATE_PREFIXES: Final = ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", +) +_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_RATE_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$") +_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX) def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool: @@ -248,9 +257,80 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str: return f"{base_key}_{suffix}" +def _parse_token_threshold(threshold: str) -> float: + return float(threshold.replace("k", "")) * (1000 if "k" in threshold else 1) + + def _parse_above_token_threshold(key: str) -> float: - threshold_str: Final = key.split("_above_")[1].split("_tokens")[0] - return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1) + return _parse_token_threshold(key.split("_above_")[1].split("_tokens")[0]) + + +def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool: + return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold) + + +@dataclass(frozen=True, slots=True) +class BatchCostRates: + input: float | None + output: float | None + cache_read: float | None + cache_creation: float | None + + +def _batch_rate(model_info: ModelInfo, key: str) -> float | None: + value: Final = model_info.get(key) + if isinstance(value, (int, float)): + return float(value) + if not isinstance(value, str): + return None + try: + return float(value) + except ValueError: + return None + + +def _batch_tier_rate(model_info: ModelInfo, tier_key: str, flat_key: str) -> float | None: + tier_rate: Final = _batch_rate(model_info, tier_key) + return _batch_rate(model_info, flat_key) if tier_rate is None else tier_rate + + +def _batch_tier_thresholds(model_info: ModelInfo, prefix: str) -> frozenset[str]: + return frozenset( + tier.group(2) + for key, value in model_info.items() + if value is not None and (tier := _BATCH_TIER_KEY.match(key)) is not None and tier.group(1) == prefix + ) + + +def _crossed_batch_tier(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> str | None: + return next( + ( + threshold + for threshold in sorted( + _batch_tier_thresholds(model_info, prefix), key=_parse_token_threshold, reverse=True + ) + if _prompt_exceeds_threshold(usage.prompt_tokens, _parse_token_threshold(threshold), inclusive) + ), + None, + ) + + +def _batch_rate_for_prefix(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> float | None: + flat_key: Final = f"{prefix}{_BATCH_KEY_SUFFIX}" + threshold: Final = _crossed_batch_tier(model_info, prefix, usage, inclusive) + if threshold is None: + return _batch_rate(model_info, flat_key) + return _batch_tier_rate(model_info, f"{prefix}_above_{threshold}_tokens{_BATCH_KEY_SUFFIX}", flat_key) + + +def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provider: str | None) -> BatchCostRates: + inclusive: Final = _uses_inclusive_token_thresholds(custom_llm_provider) + return BatchCostRates( + input=_batch_rate_for_prefix(model_info, "input_cost_per_token", usage, inclusive), + output=_batch_rate_for_prefix(model_info, "output_cost_per_token", usage, inclusive), + cache_read=_batch_rate_for_prefix(model_info, "cache_read_input_token_cost", usage, inclusive), + cache_creation=_batch_rate_for_prefix(model_info, "cache_creation_input_token_cost", usage, inclusive), + ) def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict | None: @@ -576,7 +656,9 @@ def _get_token_base_cost( # so that the threshold detection loop only processes standard keys. The # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key. threshold_keys: Final = [ - k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES) + k + for k in model_info + if k.startswith("input_cost_per_token_above_") and not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) ] # Only sort the threshold keys (typically 1-2 keys instead of 66+) @@ -588,7 +670,7 @@ def _get_token_base_cost( # Handle both formats: _above_128k_tokens and _above_128_tokens threshold_str = key.split("_above_")[1].split("_tokens")[0] threshold = _parse_above_token_threshold(key) - if usage.prompt_tokens > threshold or (threshold_is_inclusive and usage.prompt_tokens == threshold): + if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive): # Prefer a service_tier-specific above-threshold key when available, # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini # ON_DEMAND_PRIORITY. Falls back to the standard key automatically diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index 1b5a4083ebe..c40cefecdd0 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -143,7 +143,7 @@ class AzurePassthroughConfig(BasePassthroughConfig): ) -> dict: return BaseAzureLLM._base_validate_azure_environment( headers=headers, - litellm_params=GenericLiteLLMParams(**{**litellm_params, "api_key": api_key}), + litellm_params=GenericLiteLLMParams.model_validate({**litellm_params, "api_key": api_key}), ) @staticmethod diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 468d86d049d..7cde9a1162e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -34129,6 +34129,10 @@ "cache_read_input_token_cost_above_272k_tokens": 2e-06, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, + "cache_read_input_token_cost_batches": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 1e-06, + "cache_creation_input_token_cost_batches": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-05, "cache_read_input_token_cost_flex": 5e-07, "cache_read_input_token_cost_priority": 2e-06, "input_cost_per_token": 1e-05, @@ -34136,6 +34140,7 @@ "input_cost_per_token_above_272k_tokens_flex": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 4e-05, "input_cost_per_token_batches": 5e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-05, "input_cost_per_token_flex": 5e-06, "input_cost_per_token_priority": 2e-05, "litellm_provider": "openai", @@ -34148,6 +34153,7 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00015, "output_cost_per_token_batches": 2.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-05, "output_cost_per_token_flex": 2.5e-05, "output_cost_per_token_priority": 0.0001, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34199,6 +34205,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34206,6 +34216,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34218,6 +34229,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34266,6 +34278,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34273,6 +34289,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34285,6 +34302,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34335,6 +34353,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, @@ -34342,6 +34364,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, "litellm_provider": "openai", @@ -34354,6 +34377,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-06, "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, + "output_cost_per_token_above_272k_tokens_batches": 9e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34403,6 +34427,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_read_input_token_cost_batches": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-08, + "cache_creation_input_token_cost_batches": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -34410,6 +34438,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-07, "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_above_272k_tokens_batches": 2e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, "litellm_provider": "openai", @@ -34422,6 +34451,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-07, "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_above_272k_tokens_batches": 9e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34692,12 +34722,15 @@ "gpt-5.5": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34708,6 +34741,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34750,12 +34784,15 @@ "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34766,6 +34803,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34810,6 +34848,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34819,6 +34858,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34859,6 +34899,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34868,6 +34909,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34906,12 +34948,15 @@ "gpt-5.4": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34922,6 +34967,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34959,12 +35005,15 @@ "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34975,6 +35024,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -35014,6 +35064,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -35023,6 +35074,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -35064,6 +35116,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -35073,6 +35126,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { diff --git a/litellm/types/utils.py b/litellm/types/utils.py index bb0b3d2a687..f10f102d8ec 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -291,6 +291,10 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_272k_tokens_priority: float | None cache_read_input_token_cost_above_272k_tokens_flex: float | None cache_read_input_token_cost_above_512k_tokens: float | None + cache_read_input_token_cost_batches: ReadOnly[float | None] + cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] + cache_creation_input_token_cost_batches: ReadOnly[float | None] + cache_creation_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses. # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT. prompt_cache_min_tokens: int | None @@ -316,7 +320,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_second: float | None # for OpenAI Speech models input_cost_per_token_batches: float | None input_cost_per_video_token_batches: ReadOnly[float | None] + input_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token_batches: float | None + output_cost_per_token_above_272k_tokens_batches: ReadOnly[float | None] output_cost_per_token: Required[float | None] output_cost_per_token_flex: float | None # OpenAI flex service tier pricing output_cost_per_token_priority: float | None # OpenAI priority service tier pricing @@ -3718,6 +3724,10 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_read_input_token_cost_above_200k_tokens_priority: float | None = None cache_read_input_token_cost_above_272k_tokens_priority: float | None = None cache_read_input_token_cost_above_272k_tokens_flex: float | None = None + cache_read_input_token_cost_batches: float | None = None + cache_read_input_token_cost_above_272k_tokens_batches: float | None = None + cache_creation_input_token_cost_batches: float | None = None + cache_creation_input_token_cost_above_272k_tokens_batches: float | None = None cache_read_input_audio_token_cost: float | None = None cache_read_input_image_token_cost: float | None = None input_cost_per_character_above_128k_tokens: float | None = None @@ -3728,6 +3738,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): input_cost_per_token_above_200k_tokens_priority: float | None = None input_cost_per_token_above_272k_tokens_priority: float | None = None input_cost_per_token_above_272k_tokens_flex: float | None = None + input_cost_per_token_above_272k_tokens_batches: float | None = None input_cost_per_query: float | None = None input_cost_per_image: float | None = None input_cost_per_image_above_128k_tokens: float | None = None @@ -3751,6 +3762,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_token_above_200k_tokens_priority: float | None = None output_cost_per_token_above_272k_tokens_priority: float | None = None output_cost_per_token_above_272k_tokens_flex: float | None = None + output_cost_per_token_above_272k_tokens_batches: float | None = None output_cost_per_character_above_128k_tokens: float | None = None output_cost_per_image: float | None = None output_cost_per_image_token: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index df2a6adbe1f..8ade6022e3e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6042,6 +6042,14 @@ def _get_model_info_helper( cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None), cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None), cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None), + cache_read_input_token_cost_batches=_model_info.get("cache_read_input_token_cost_batches"), + cache_read_input_token_cost_above_272k_tokens_batches=_model_info.get( + "cache_read_input_token_cost_above_272k_tokens_batches" + ), + cache_creation_input_token_cost_batches=_model_info.get("cache_creation_input_token_cost_batches"), + cache_creation_input_token_cost_above_272k_tokens_batches=_model_info.get( + "cache_creation_input_token_cost_above_272k_tokens_batches" + ), cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), @@ -6072,7 +6080,13 @@ def _get_model_info_helper( input_cost_per_video_per_second=_model_info.get("input_cost_per_video_per_second", None), input_cost_per_token_batches=_model_info.get("input_cost_per_token_batches"), input_cost_per_video_token_batches=_model_info.get("input_cost_per_video_token_batches", None), + input_cost_per_token_above_272k_tokens_batches=_model_info.get( + "input_cost_per_token_above_272k_tokens_batches" + ), output_cost_per_token_batches=_model_info.get("output_cost_per_token_batches"), + output_cost_per_token_above_272k_tokens_batches=_model_info.get( + "output_cost_per_token_above_272k_tokens_batches" + ), output_cost_per_token=_output_cost_per_token, output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None), output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 468d86d049d..7cde9a1162e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -34129,6 +34129,10 @@ "cache_read_input_token_cost_above_272k_tokens": 2e-06, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06, + "cache_read_input_token_cost_batches": 5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 1e-06, + "cache_creation_input_token_cost_batches": 6.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 1.25e-05, "cache_read_input_token_cost_flex": 5e-07, "cache_read_input_token_cost_priority": 2e-06, "input_cost_per_token": 1e-05, @@ -34136,6 +34140,7 @@ "input_cost_per_token_above_272k_tokens_flex": 1e-05, "input_cost_per_token_above_272k_tokens_priority": 4e-05, "input_cost_per_token_batches": 5e-06, + "input_cost_per_token_above_272k_tokens_batches": 1e-05, "input_cost_per_token_flex": 5e-06, "input_cost_per_token_priority": 2e-05, "litellm_provider": "openai", @@ -34148,6 +34153,7 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-05, "output_cost_per_token_above_272k_tokens_priority": 0.00015, "output_cost_per_token_batches": 2.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 3.75e-05, "output_cost_per_token_flex": 2.5e-05, "output_cost_per_token_priority": 0.0001, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34199,6 +34205,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34206,6 +34216,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34218,6 +34229,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34266,6 +34278,10 @@ "cache_read_input_token_cost_above_272k_tokens": 8e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 1.6e-06, + "cache_read_input_token_cost_batches": 2e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 4e-07, + "cache_creation_input_token_cost_batches": 2.5e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 5e-06, "cache_read_input_token_cost_flex": 2e-07, "cache_read_input_token_cost_priority": 8e-07, "input_cost_per_token": 4e-06, @@ -34273,6 +34289,7 @@ "input_cost_per_token_above_272k_tokens_flex": 4e-06, "input_cost_per_token_above_272k_tokens_priority": 1.6e-05, "input_cost_per_token_batches": 2e-06, + "input_cost_per_token_above_272k_tokens_batches": 4e-06, "input_cost_per_token_flex": 2e-06, "input_cost_per_token_priority": 8e-06, "litellm_provider": "openai", @@ -34285,6 +34302,7 @@ "output_cost_per_token_above_272k_tokens_flex": 1.5e-05, "output_cost_per_token_above_272k_tokens_priority": 6e-05, "output_cost_per_token_batches": 1e-05, + "output_cost_per_token_above_272k_tokens_batches": 1.5e-05, "output_cost_per_token_flex": 1e-05, "output_cost_per_token_priority": 4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34335,6 +34353,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-07, + "cache_read_input_token_cost_batches": 1e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-07, + "cache_creation_input_token_cost_batches": 1.25e-06, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 4e-07, "input_cost_per_token": 2e-06, @@ -34342,6 +34364,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, "litellm_provider": "openai", @@ -34354,6 +34377,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-06, "output_cost_per_token_above_272k_tokens_priority": 3.6e-05, "output_cost_per_token_batches": 6e-06, + "output_cost_per_token_above_272k_tokens_batches": 9e-06, "output_cost_per_token_flex": 6e-06, "output_cost_per_token_priority": 2.4e-05, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34403,6 +34427,10 @@ "cache_read_input_token_cost_above_272k_tokens": 4e-08, "cache_read_input_token_cost_above_272k_tokens_flex": 2e-08, "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08, + "cache_read_input_token_cost_batches": 1e-08, + "cache_read_input_token_cost_above_272k_tokens_batches": 2e-08, + "cache_creation_input_token_cost_batches": 1.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1e-08, "cache_read_input_token_cost_priority": 4e-08, "input_cost_per_token": 2e-07, @@ -34410,6 +34438,7 @@ "input_cost_per_token_above_272k_tokens_flex": 2e-07, "input_cost_per_token_above_272k_tokens_priority": 8e-07, "input_cost_per_token_batches": 1e-07, + "input_cost_per_token_above_272k_tokens_batches": 2e-07, "input_cost_per_token_flex": 1e-07, "input_cost_per_token_priority": 4e-07, "litellm_provider": "openai", @@ -34422,6 +34451,7 @@ "output_cost_per_token_above_272k_tokens_flex": 9e-07, "output_cost_per_token_above_272k_tokens_priority": 3.6e-06, "output_cost_per_token_batches": 6e-07, + "output_cost_per_token_above_272k_tokens_batches": 9e-07, "output_cost_per_token_flex": 6e-07, "output_cost_per_token_priority": 2.4e-06, "regional_processing_uplift_multiplier_eu": 1.1, @@ -34692,12 +34722,15 @@ "gpt-5.5": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34708,6 +34741,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34750,12 +34784,15 @@ "gpt-5.5-2026-04-23": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_above_272k_tokens": 1e-06, + "cache_read_input_token_cost_batches": 2.5e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 5e-07, "cache_read_input_token_cost_flex": 2.5e-07, "cache_read_input_token_cost_priority": 1.25e-06, "input_cost_per_token": 5e-06, "input_cost_per_token_above_272k_tokens": 1e-05, "input_cost_per_token_flex": 2.5e-06, "input_cost_per_token_batches": 2.5e-06, + "input_cost_per_token_above_272k_tokens_batches": 5e-06, "input_cost_per_token_priority": 1.25e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34766,6 +34803,7 @@ "output_cost_per_token_above_272k_tokens": 4.5e-05, "output_cost_per_token_flex": 1.5e-05, "output_cost_per_token_batches": 1.5e-05, + "output_cost_per_token_above_272k_tokens_batches": 2.25e-05, "output_cost_per_token_priority": 7.5e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34810,6 +34848,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34819,6 +34858,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34859,6 +34899,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -34868,6 +34909,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -34906,12 +34948,15 @@ "gpt-5.4": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34922,6 +34967,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -34959,12 +35005,15 @@ "gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, "cache_read_input_token_cost_above_272k_tokens": 5e-07, + "cache_read_input_token_cost_batches": 1.25e-07, + "cache_read_input_token_cost_above_272k_tokens_batches": 2.5e-07, "cache_read_input_token_cost_flex": 1.3e-07, "cache_read_input_token_cost_priority": 5e-07, "input_cost_per_token": 2.5e-06, "input_cost_per_token_above_272k_tokens": 5e-06, "input_cost_per_token_flex": 1.25e-06, "input_cost_per_token_batches": 1.25e-06, + "input_cost_per_token_above_272k_tokens_batches": 2.5e-06, "input_cost_per_token_priority": 5e-06, "litellm_provider": "openai", "max_input_tokens": 1050000, @@ -34975,6 +35024,7 @@ "output_cost_per_token_above_272k_tokens": 2.25e-05, "output_cost_per_token_flex": 7.5e-06, "output_cost_per_token_batches": 7.5e-06, + "output_cost_per_token_above_272k_tokens_batches": 1.125e-05, "output_cost_per_token_priority": 3e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, @@ -35014,6 +35064,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -35023,6 +35074,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { @@ -35064,6 +35116,7 @@ "input_cost_per_token_above_272k_tokens": 6e-05, "input_cost_per_token_flex": 1.5e-05, "input_cost_per_token_batches": 1.5e-05, + "input_cost_per_token_above_272k_tokens_batches": 3e-05, "litellm_provider": "openai", "max_input_tokens": 1050000, "max_output_tokens": 128000, @@ -35073,6 +35126,7 @@ "output_cost_per_token_above_272k_tokens": 0.00027, "output_cost_per_token_flex": 9e-05, "output_cost_per_token_batches": 9e-05, + "output_cost_per_token_above_272k_tokens_batches": 0.000135, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 16b762f7803..737a9b7fa60 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -113,6 +113,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_creation_input_token_cost_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_creation_input_token_cost_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -123,6 +128,10 @@ "minimum": 0, "description": "Priority service-tier rate for the same-named base field." }, + "cache_creation_input_token_cost_batches": { + "type": "number", + "minimum": 0 + }, "cache_creation_input_token_cost_flex": { "type": "number", "minimum": 0, @@ -171,6 +180,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_read_input_token_cost_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "cache_read_input_token_cost_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -186,6 +200,10 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_read_input_token_cost_batches": { + "type": "number", + "minimum": 0 + }, "cache_read_input_token_cost_flex": { "type": "number", "minimum": 0, @@ -338,6 +356,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "input_cost_per_token_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "input_cost_per_token_above_272k_tokens_flex": { "type": "number", "minimum": 0, @@ -669,6 +692,11 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "output_cost_per_token_above_272k_tokens_batches": { + "type": "number", + "minimum": 0, + "description": "Rate applied once the prompt exceeds the token threshold in the field name." + }, "output_cost_per_token_above_272k_tokens_flex": { "type": "number", "minimum": 0, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 4fe3d410ef8..4b25c87d70f 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,5 +1,6 @@ from collections.abc import Mapping from datetime import datetime, timezone +from typing import cast import pytest @@ -3634,3 +3635,147 @@ def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_ assert creation == pytest.approx(expected_creation) assert creation_1h == pytest.approx(expected_creation_1h) + + +def _batch_rates_model_info(**rates: object) -> ModelInfo: + return cast(ModelInfo, dict(rates)) + + +def test_get_batch_cost_rates_parses_string_rates(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches="1e-06", + output_cost_per_token_batches="4e-06", + cache_read_input_token_cost_batches="1e-07", + input_cost_per_token_above_272k_tokens_batches="2e-06", + output_cost_per_token_above_272k_tokens_batches="6e-06", + cache_read_input_token_cost_above_272k_tokens_batches="2e-07", + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert (rates.input, rates.output, rates.cache_read) == (2e-6, 6e-6, 2e-7) + + +def test_get_batch_cost_rates_falls_back_to_the_flat_rates_when_tier_rates_are_unparsable(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost_batches=1e-7, + input_cost_per_token_above_272k_tokens_batches="two", + output_cost_per_token_above_272k_tokens_batches="six", + cache_read_input_token_cost_above_272k_tokens_batches="none", + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_272k_tokens_batches="nope", + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert (rates.input, rates.output, rates.cache_read, rates.cache_creation) == (1e-6, 4e-6, 1e-7, 1.25e-7) + + +def test_get_batch_cost_rates_has_no_cached_rate_without_a_cached_batch_key(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost=2e-7, + input_cost_per_token_above_272k_tokens_batches=2e-6, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert (rates.input, rates.output, rates.cache_read) == (2e-6, 4e-6, None) + + +@pytest.mark.parametrize(("prompt_tokens", "expected"), [(1_000, 1.25e-7), (272_000, 1.25e-7), (300_000, 2.5e-7)]) +def test_get_batch_cost_rates_reads_the_batch_cache_write_rate_for_the_crossed_tier(prompt_tokens, expected): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-7, + input_cost_per_token_above_272k_tokens_batches=2e-7, + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_272k_tokens_batches=2.5e-7, + ), + Usage(prompt_tokens=prompt_tokens, completion_tokens=1, total_tokens=prompt_tokens + 1), + "openai", + ) + + assert rates.cache_creation == expected + + +@pytest.mark.parametrize( + ("tier_key", "attribute"), + [ + ("output_cost_per_token_above_272k_tokens_batches", "output"), + ("cache_read_input_token_cost_above_272k_tokens_batches", "cache_read"), + ("cache_creation_input_token_cost_above_272k_tokens_batches", "cache_creation"), + ], +) +def test_get_batch_cost_rates_crosses_a_tier_declared_without_an_input_tier_key(tier_key, attribute): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost_batches=1e-7, + cache_creation_input_token_cost_batches=1.25e-7, + **{tier_key: 9e-6}, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert getattr(rates, attribute) == 9e-6 + assert rates.input == 1e-6 + + +@pytest.mark.parametrize( + ("prompt_tokens", "expected_input", "expected_output"), + [(200_000, 1e-6, 4e-6), (250_000, 1e-6, 5e-6), (300_000, 2e-6, 5e-6)], +) +def test_get_batch_cost_rates_crosses_each_components_own_tier(prompt_tokens, expected_input, expected_output): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_batches=4e-6, + output_cost_per_token_above_200k_tokens_batches=5e-6, + ), + Usage(prompt_tokens=prompt_tokens, completion_tokens=1, total_tokens=prompt_tokens + 1), + "openai", + ) + + assert (rates.input, rates.output) == (expected_input, expected_output) + + +def test_get_batch_cost_rates_has_no_cache_write_rate_without_a_cache_write_batch_key(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=1e-7, + input_cost_per_token_above_272k_tokens_batches=2e-7, + cache_creation_input_token_cost=2.5e-7, + cache_creation_input_token_cost_above_272k_tokens=5e-7, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert rates.cache_creation is None diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 52eed9c9f95..b0fc725b50a 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -6,6 +6,7 @@ import logging import os import sys from collections.abc import Callable, Iterator, Mapping +from types import MappingProxyType from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -7478,6 +7479,141 @@ def test_passthrough_embeddings_result_swapped_for_callbacks(): assert swapped_result.data[0]["embedding"] == [0.1, 0.2, 0.3] +_PUBLISHED_BATCH_MODEL: Final = "lit-published-batch-tier-model" +_PUBLISHED_BATCH_DEPLOYMENT: Final = f"openai/{_PUBLISHED_BATCH_MODEL}" +_PUBLISHED_BATCH_RATES: Final = MappingProxyType( + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "input_cost_per_token_batches": 1.1e-6, + "output_cost_per_token_batches": 4.1e-6, + "cache_read_input_token_cost_batches": 1.2e-7, + "cache_creation_input_token_cost_batches": 1.3e-6, + "input_cost_per_token_above_272k_tokens_batches": 3.1e-6, + "output_cost_per_token_above_272k_tokens_batches": 7.1e-6, + "cache_read_input_token_cost_above_272k_tokens_batches": 3.2e-7, + "cache_creation_input_token_cost_above_272k_tokens_batches": 3.3e-6, + } +) +_PUBLISHED_INPUT_BATCH_KEYS: Final = ( + "input_cost_per_token_batches", + "input_cost_per_token_above_272k_tokens_batches", + "cache_read_input_token_cost_batches", + "cache_read_input_token_cost_above_272k_tokens_batches", + "cache_creation_input_token_cost_batches", + "cache_creation_input_token_cost_above_272k_tokens_batches", +) +_PUBLISHED_OUTPUT_BATCH_KEYS: Final = ( + "output_cost_per_token_batches", + "output_cost_per_token_above_272k_tokens_batches", +) + + +@pytest.fixture +def _published_batch_model(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.register_model( + model_cost={_PUBLISHED_BATCH_MODEL: {**_PUBLISHED_BATCH_RATES}}, persist_across_reloads=False + ) + + +def _batch_deployment_id(custom_pricing: dict[str, float]) -> str: + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "published-batch", + "litellm_params": {"model": _PUBLISHED_BATCH_DEPLOYMENT, "api_key": "sk-test", **custom_pricing}, + } + ] + ) + return router.model_list[0]["model_info"]["id"] + + +def test_deployment_pricing_model_info_carries_every_published_input_batch_rate_when_only_output_is_declared( + _published_batch_model: None, +) -> None: + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"output_cost_per_token_batches": 4e-6}), _PUBLISHED_BATCH_DEPLOYMENT + ) + + assert info is not None + assert {key: info[key] for key in _PUBLISHED_INPUT_BATCH_KEYS} == { + key: _PUBLISHED_BATCH_RATES[key] for key in _PUBLISHED_INPUT_BATCH_KEYS + } + assert info["output_cost_per_token_batches"] == 4e-6 + assert info["output_cost_per_token_above_272k_tokens_batches"] is None + + +def test_deployment_pricing_model_info_carries_the_published_output_batch_tier_when_only_input_is_declared( + _published_batch_model: None, +) -> None: + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"input_cost_per_token_batches": 1e-6}), _PUBLISHED_BATCH_DEPLOYMENT + ) + + assert info is not None + assert {key: info[key] for key in _PUBLISHED_OUTPUT_BATCH_KEYS} == { + key: _PUBLISHED_BATCH_RATES[key] for key in _PUBLISHED_OUTPUT_BATCH_KEYS + } + assert info["input_cost_per_token_batches"] == 1e-6 + assert info["input_cost_per_token_above_272k_tokens_batches"] is None + assert info["cache_read_input_token_cost_batches"] is None + assert info["cache_creation_input_token_cost_batches"] is None + + +def test_batch_cost_calculator_bills_the_carried_output_tier_when_the_deployment_declares_its_own_input_rate( + _published_batch_model: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + from litellm.types.utils import Usage + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"input_cost_per_token": 5e-6}), _PUBLISHED_BATCH_DEPLOYMENT + ) + assert info is not None + + prompt_cost, completion_cost = batch_cost_calculator( + usage=Usage(prompt_tokens=300_000, completion_tokens=10, total_tokens=300_010), + model=_PUBLISHED_BATCH_DEPLOYMENT, + custom_llm_provider="openai", + model_info=info, + ) + + assert prompt_cost == pytest.approx(300_000 * 5e-6 / 2) + assert completion_cost == pytest.approx( + 10 * _PUBLISHED_BATCH_RATES["output_cost_per_token_above_272k_tokens_batches"] + ) + + +def test_deployment_pricing_model_info_honors_a_tier_only_batch_override_over_the_published_flat_rates( + _published_batch_model: None, +) -> None: + from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info + + info: Final = deployment_pricing_model_info( + _batch_deployment_id({"input_cost_per_token_above_272k_tokens_batches": 1e-3}), _PUBLISHED_BATCH_DEPLOYMENT + ) + carried_keys: Final = tuple( + key + for key in (*_PUBLISHED_INPUT_BATCH_KEYS, *_PUBLISHED_OUTPUT_BATCH_KEYS) + if key != "input_cost_per_token_above_272k_tokens_batches" + ) + + assert info is not None + assert info["input_cost_per_token_above_272k_tokens_batches"] == 1e-3 + assert {key: info[key] for key in carried_keys} == {key: _PUBLISHED_BATCH_RATES[key] for key in carried_keys} + + def test_get_status_fields_ranks_guardrail_flagged_between_success_and_intervened(): """LIT-6894: a non-blocking flagged verdict must outrank success in the request-level guardrail_status but never mask an intervention.""" diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index bdeedb5f38c..a33701cd1bf 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,6 +1,7 @@ import datetime import time -from typing import Final +from types import MappingProxyType +from typing import Final, cast import pytest from pydantic import BaseModel @@ -4464,6 +4465,273 @@ def test_completion_cost_prices_responses_websocket_turns_per_service_tier(): assert ws_cost != pytest.approx(_http_cost(160, 50, "priority")) +_TIERED_BATCH_MODEL: Final = "lit-tiered-batch-model" +_FLAT_CACHE_BATCH_MODEL: Final = "lit-tiered-batch-model-without-cache-batch-rates" +_TIERED_BATCH_ENTRY: Final = MappingProxyType( + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "cache_read_input_token_cost": 2e-7, + "cache_creation_input_token_cost": 2.5e-6, + "input_cost_per_token_above_272k_tokens": 4e-6, + "output_cost_per_token_above_272k_tokens": 1.2e-5, + "input_cost_per_token_batches": 1e-6, + "output_cost_per_token_batches": 4e-6, + "cache_read_input_token_cost_batches": 1e-7, + "cache_creation_input_token_cost_batches": 1.25e-6, + "input_cost_per_token_above_272k_tokens_batches": 3e-6, + "output_cost_per_token_above_272k_tokens_batches": 7e-6, + "cache_read_input_token_cost_above_272k_tokens_batches": 3e-7, + "cache_creation_input_token_cost_above_272k_tokens_batches": 3.75e-6, + } +) +_BATCH_RATE_PREFIXES: Final = ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", +) + + +@pytest.fixture +def _tiered_batch_models(_local_model_cost_map: None) -> None: + litellm.register_model( + model_cost={ + _TIERED_BATCH_MODEL: {**_TIERED_BATCH_ENTRY}, + _FLAT_CACHE_BATCH_MODEL: { + key: rate + for key, rate in _TIERED_BATCH_ENTRY.items() + if not (key.startswith("cache_") and key.endswith("_batches")) + }, + }, + persist_across_reloads=False, + ) + + +def test_batch_cost_calculator_bills_the_long_context_batch_tier_above_272k(_tiered_batch_models: None) -> None: + from litellm.cost_calculator import batch_cost_calculator + + usage: Final = Usage(prompt_tokens=300_035, completion_tokens=64, total_tokens=300_099) + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(300_035 * 3e-6) + assert completion_cost_value == pytest.approx(64 * 7e-6) + + +@pytest.mark.parametrize("prompt_tokens", [272_000, 1_000]) +def test_batch_cost_calculator_bills_the_flat_batch_rate_at_or_below_272k( + _tiered_batch_models: None, prompt_tokens: int +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=64, total_tokens=prompt_tokens + 64) + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(prompt_tokens * 1e-6) + assert completion_cost_value == pytest.approx(64 * 4e-6) + + +def test_get_model_info_exposes_every_registered_batch_rate(_tiered_batch_models: None) -> None: + info: Final = litellm.get_model_info(_TIERED_BATCH_MODEL, custom_llm_provider="openai") + batch_keys: Final = tuple(key for key in _TIERED_BATCH_ENTRY if key.endswith("_batches")) + + assert len(batch_keys) == 8 + assert {key: info[key] for key in batch_keys} == {key: _TIERED_BATCH_ENTRY[key] for key in batch_keys} + + +def test_regular_path_never_bills_the_batch_tier_keys(_local_model_cost_map, monkeypatch): + monkeypatch.setitem( + litellm.model_cost, + "lit-batch-tier-guard", + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "input_cost_per_token_batches": 1e-6, + "output_cost_per_token_batches": 4e-6, + "input_cost_per_token_above_272k_tokens_batches": 5e-6, + "output_cost_per_token_above_272k_tokens_batches": 9e-6, + }, + ) + + prompt_cost, completion_cost = litellm.cost_per_token( + model="lit-batch-tier-guard", custom_llm_provider="openai", prompt_tokens=300_035, completion_tokens=64 + ) + + assert prompt_cost == pytest.approx(300_035 * 2e-6) + assert completion_cost == pytest.approx(64 * 8e-6) + + +@pytest.mark.parametrize("prefix", _BATCH_RATE_PREFIXES) +def test_every_openai_entry_with_a_long_context_rate_and_a_batch_rate_declares_the_batch_tier( + _local_model_cost_map: None, prefix: str +) -> None: + undeclared: Final = [ + name + for name, entry in litellm.model_cost.items() + if isinstance(entry, dict) + and entry.get("litellm_provider") == "openai" + and entry.get(f"{prefix}_above_272k_tokens") is not None + and entry.get(f"{prefix}_batches") is not None + and entry.get(f"{prefix}_above_272k_tokens_batches") is None + ] + + assert undeclared == [] + + +def test_batch_cost_calculator_ignores_malformed_batch_tier_keys(): + from litellm.cost_calculator import batch_cost_calculator + + usage = Usage(prompt_tokens=300_035, completion_tokens=64, total_tokens=300_099) + model_info = cast( + ModelInfo, + { + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "input_cost_per_token_batches": 1e-6, + "output_cost_per_token_batches": 4e-6, + "input_cost_per_token_above_272k_tokens_batches": 2e-6, + "output_cost_per_token_above_272k_tokens_batches": 6e-6, + "input_cost_per_token_above_lots_tokens_batches": 1.0, + }, + ) + + prompt_cost, completion_cost = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(300_035 * 2e-6) + assert completion_cost == pytest.approx(64 * 6e-6) + + +def _cached_usage(prompt_tokens: int, cached_tokens: int, completion_tokens: int) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens), + ) + + +def test_batch_cost_calculator_bills_cached_tokens_at_the_long_context_batch_cached_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=_cached_usage(300_048, 300_045, 11), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(3 * 3e-6 + 300_045 * 3e-7) + assert completion_cost_value == pytest.approx(11 * 7e-6) + + +def test_batch_cost_calculator_bills_cached_tokens_at_the_flat_batch_cached_rate_at_or_below_272k( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cached_usage(1_000, 900, 4), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(100 * 1e-6 + 900 * 1e-7) + + +def test_batch_cost_calculator_bills_cached_tokens_at_the_batch_input_rate_without_a_cached_batch_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cached_usage(300_048, 300_045, 11), model=_FLAT_CACHE_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(300_048 * 3e-6) + + +def _cache_write_usage(prompt_tokens: int, cache_write_tokens: int, completion_tokens: int) -> Usage: + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=0, cache_write_tokens=cache_write_tokens), + ) + + +def test_batch_cost_calculator_bills_cache_write_tokens_at_the_long_context_batch_cache_write_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=_cache_write_usage(300_048, 300_045, 4), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(3 * 3e-6 + 300_045 * 3.75e-6) + assert completion_cost_value == pytest.approx(4 * 7e-6) + + +def test_batch_cost_calculator_bills_cache_write_tokens_at_the_flat_batch_cache_write_rate_at_or_below_272k( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cache_write_usage(1_000, 900, 4), model=_TIERED_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(100 * 1e-6 + 900 * 1.25e-6) + + +def test_batch_cost_calculator_bills_cache_write_tokens_at_the_batch_input_rate_without_a_cache_write_batch_rate( + _tiered_batch_models: None, +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + prompt_cost, _ = batch_cost_calculator( + usage=_cache_write_usage(300_048, 300_045, 4), model=_FLAT_CACHE_BATCH_MODEL, custom_llm_provider="openai" + ) + + assert prompt_cost == pytest.approx(300_048 * 3e-6) + + +def test_batch_cost_calculator_prices_modalities_and_cached_tokens_together_in_the_crossed_tier() -> None: + from litellm.cost_calculator import batch_cost_calculator + + model_info: Final = cast( + ModelInfo, + { + "input_cost_per_token_batches": 1e-6, + "input_cost_per_token_above_272k_tokens_batches": 3e-6, + "input_cost_per_audio_token_batches": 5e-6, + "cache_read_input_token_cost_batches": 1e-7, + "cache_read_input_token_cost_above_272k_tokens_batches": 3e-7, + }, + ) + usage: Final = Usage( + prompt_tokens=300_000, + completion_tokens=0, + total_tokens=300_000, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=64, image_tokens=10, cached_tokens=1_000), + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model=_TIERED_BATCH_MODEL, custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(298_926 * 3e-6 + 64 * 5e-6 + 10 * 3e-6 + 1_000 * 3e-7) + + QWEN3_NEXT_REGIONS: Final = ("ap-northeast-1", "ap-south-1", "ap-southeast-2", "eu-west-1", "eu-west-2", "sa-east-1") diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 38aad3ace1c..96c420299e7 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -740,6 +740,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_creation_input_token_cost_above_272k_tokens": {"type": "number"}, "cache_creation_input_token_cost_above_272k_tokens_flex": {"type": "number"}, "cache_creation_input_token_cost_above_272k_tokens_priority": {"type": "number"}, + "cache_creation_input_token_cost_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_batches": {"type": "number"}, "cache_creation_input_token_cost_flex": {"type": "number"}, "cache_creation_input_token_cost_priority": {"type": "number"}, "cache_read_input_token_cost": {"type": "number"}, @@ -749,6 +751,8 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "cache_read_input_token_cost_above_272k_tokens": {"type": "number"}, "cache_read_input_token_cost_above_272k_tokens_flex": {"type": "number"}, "cache_read_input_token_cost_above_512k_tokens": {"type": "number"}, + "cache_read_input_token_cost_batches": {"type": "number"}, + "cache_read_input_token_cost_above_272k_tokens_batches": {"type": "number"}, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, "cache_read_input_image_token_cost": {"type": "number"}, @@ -776,12 +780,14 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_token_priority": {"type": "number"}, "input_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "input_cost_per_token_above_272k_tokens_priority": {"type": "number"}, + "input_cost_per_token_above_272k_tokens_batches": {"type": "number"}, "input_cost_per_token_above_272k_tokens_flex": {"type": "number"}, "input_cost_per_audio_token_priority": {"type": "number"}, "output_cost_per_token_flex": {"type": "number"}, "output_cost_per_token_priority": {"type": "number"}, "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "output_cost_per_token_above_272k_tokens_priority": {"type": "number"}, + "output_cost_per_token_above_272k_tokens_batches": {"type": "number"}, "output_cost_per_token_above_272k_tokens_flex": {"type": "number"}, "regional_endpoint_uplift_multiplier": {"type": "number"}, "regional_processing_uplift_multiplier_eu": {"type": "number"}, diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index 708c472939e..a4de8eee23c 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -26,7 +26,7 @@ from openai.types.batch import BatchRequestCounts import litellm import litellm.batches.batch_utils as bu -from litellm.types.utils import LiteLLMBatch, Usage +from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage # --------------------------------------------------------------------------- # # Builders for batch OUTPUT file rows. @@ -437,6 +437,33 @@ def test_total_usage_and_cost_normalize_mixed_responses_and_chat(): assert result.cost == pytest.approx((30 * 0.00125) + (12 * 0.005)) +def test_total_cost_applies_the_long_context_batch_tier_per_line(): + long_row = _success_row(usage=_usage(300_000, 10)) + short_row = _success_row(usage=_usage(100, 10)) + + result = bu._aggregate_batch_cost_usage_models( + entries=[long_row, short_row], + custom_llm_provider="openai", + model_info=ModelInfo( + key="lit-batch-tier", + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=2e-6, + output_cost_per_token=8e-6, + litellm_provider="openai", + mode="chat", + supported_openai_params=None, + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_above_272k_tokens_batches=6e-6, + ), + ) + + assert result.cost == pytest.approx((300_000 * 2e-6) + (10 * 6e-6) + (100 * 1e-6) + (10 * 4e-6)) + + def test_total_usage_empty_is_zero(): result = bu._aggregate_batch_cost_usage_models(entries=[], custom_llm_provider="openai") assert result.cost == 0.0 @@ -1824,6 +1851,45 @@ def test_unparsable_bedrock_batch_usage_warns(caplog): assert "inputTextTokenCount" in caplog.text +def test_total_cost_bills_cached_tokens_per_line_at_the_batch_cached_rate(): + responses_row = _success_row( + usage={ + "input_tokens": 300_000, + "output_tokens": 10, + "total_tokens": 300_010, + "input_tokens_details": {"cached_tokens": 299_000}, + } + ) + chat_row = _success_row(usage={**_usage(100, 10), "prompt_tokens_details": {"cached_tokens": 60}}) + + result = bu._aggregate_batch_cost_usage_models( + entries=[responses_row, chat_row], + custom_llm_provider="openai", + model_info=ModelInfo( + key="lit-batch-cached-tier", + max_tokens=None, + max_input_tokens=None, + max_output_tokens=None, + input_cost_per_token=2e-6, + output_cost_per_token=8e-6, + cache_read_input_token_cost=1e-6, + litellm_provider="openai", + mode="chat", + supported_openai_params=None, + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost_batches=5e-7, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_above_272k_tokens_batches=6e-6, + cache_read_input_token_cost_above_272k_tokens_batches=1e-6, + ), + ) + + long_line = 1_000 * 2e-6 + 299_000 * 1e-6 + 10 * 6e-6 + short_line = 40 * 1e-6 + 60 * 5e-7 + 10 * 4e-6 + assert result.cost == pytest.approx(long_line + short_line) + + # --------------------------------------------------------------------------- # # batch_cost_is_final # --------------------------------------------------------------------------- # diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 38cfdc9666b..38f8ac365ff 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -31186,10 +31186,14 @@ export interface components { cache_creation_input_token_cost_above_200k_tokens?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens */ cache_creation_input_token_cost_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 272K Tokens Batches */ + cache_creation_input_token_cost_above_272k_tokens_batches?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens Flex */ cache_creation_input_token_cost_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens Priority */ cache_creation_input_token_cost_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Batches */ + cache_creation_input_token_cost_batches?: number | null; /** Cache Creation Input Token Cost Flex */ cache_creation_input_token_cost_flex?: number | null; /** Cache Creation Input Token Cost Priority */ @@ -31208,12 +31212,16 @@ export interface components { cache_read_input_token_cost_above_200k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 272K Tokens */ cache_read_input_token_cost_above_272k_tokens?: number | null; + /** Cache Read Input Token Cost Above 272K Tokens Batches */ + cache_read_input_token_cost_above_272k_tokens_batches?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Flex */ cache_read_input_token_cost_above_272k_tokens_flex?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Priority */ cache_read_input_token_cost_above_272k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 512K Tokens */ cache_read_input_token_cost_above_512k_tokens?: number | null; + /** Cache Read Input Token Cost Batches */ + cache_read_input_token_cost_batches?: number | null; /** Cache Read Input Token Cost Flex */ cache_read_input_token_cost_flex?: number | null; /** Cache Read Input Token Cost Priority */ @@ -31282,6 +31290,8 @@ export interface components { input_cost_per_token_above_200k_tokens_priority?: number | null; /** Input Cost Per Token Above 272K Tokens */ input_cost_per_token_above_272k_tokens?: number | null; + /** Input Cost Per Token Above 272K Tokens Batches */ + input_cost_per_token_above_272k_tokens_batches?: number | null; /** Input Cost Per Token Above 272K Tokens Flex */ input_cost_per_token_above_272k_tokens_flex?: number | null; /** Input Cost Per Token Above 272K Tokens Priority */ @@ -31403,6 +31413,8 @@ export interface components { output_cost_per_token_above_200k_tokens_priority?: number | null; /** Output Cost Per Token Above 272K Tokens */ output_cost_per_token_above_272k_tokens?: number | null; + /** Output Cost Per Token Above 272K Tokens Batches */ + output_cost_per_token_above_272k_tokens_batches?: number | null; /** Output Cost Per Token Above 272K Tokens Flex */ output_cost_per_token_above_272k_tokens_flex?: number | null; /** Output Cost Per Token Above 272K Tokens Priority */ @@ -42026,10 +42038,14 @@ export interface components { cache_creation_input_token_cost_above_200k_tokens?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens */ cache_creation_input_token_cost_above_272k_tokens?: number | null; + /** Cache Creation Input Token Cost Above 272K Tokens Batches */ + cache_creation_input_token_cost_above_272k_tokens_batches?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens Flex */ cache_creation_input_token_cost_above_272k_tokens_flex?: number | null; /** Cache Creation Input Token Cost Above 272K Tokens Priority */ cache_creation_input_token_cost_above_272k_tokens_priority?: number | null; + /** Cache Creation Input Token Cost Batches */ + cache_creation_input_token_cost_batches?: number | null; /** Cache Creation Input Token Cost Flex */ cache_creation_input_token_cost_flex?: number | null; /** Cache Creation Input Token Cost Priority */ @@ -42048,12 +42064,16 @@ export interface components { cache_read_input_token_cost_above_200k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 272K Tokens */ cache_read_input_token_cost_above_272k_tokens?: number | null; + /** Cache Read Input Token Cost Above 272K Tokens Batches */ + cache_read_input_token_cost_above_272k_tokens_batches?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Flex */ cache_read_input_token_cost_above_272k_tokens_flex?: number | null; /** Cache Read Input Token Cost Above 272K Tokens Priority */ cache_read_input_token_cost_above_272k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 512K Tokens */ cache_read_input_token_cost_above_512k_tokens?: number | null; + /** Cache Read Input Token Cost Batches */ + cache_read_input_token_cost_batches?: number | null; /** Cache Read Input Token Cost Flex */ cache_read_input_token_cost_flex?: number | null; /** Cache Read Input Token Cost Priority */ @@ -42122,6 +42142,8 @@ export interface components { input_cost_per_token_above_200k_tokens_priority?: number | null; /** Input Cost Per Token Above 272K Tokens */ input_cost_per_token_above_272k_tokens?: number | null; + /** Input Cost Per Token Above 272K Tokens Batches */ + input_cost_per_token_above_272k_tokens_batches?: number | null; /** Input Cost Per Token Above 272K Tokens Flex */ input_cost_per_token_above_272k_tokens_flex?: number | null; /** Input Cost Per Token Above 272K Tokens Priority */ @@ -42243,6 +42265,8 @@ export interface components { output_cost_per_token_above_200k_tokens_priority?: number | null; /** Output Cost Per Token Above 272K Tokens */ output_cost_per_token_above_272k_tokens?: number | null; + /** Output Cost Per Token Above 272K Tokens Batches */ + output_cost_per_token_above_272k_tokens_batches?: number | null; /** Output Cost Per Token Above 272K Tokens Flex */ output_cost_per_token_above_272k_tokens_flex?: number | null; /** Output Cost Per Token Above 272K Tokens Priority */