diff --git a/litellm/_catalog/__init__.py b/litellm/_catalog/__init__.py index c852d4c3125..70ae7339ad5 100644 --- a/litellm/_catalog/__init__.py +++ b/litellm/_catalog/__init__.py @@ -15,6 +15,12 @@ class Reloaded: models_count: int +def is_priced(model: str, provider: str | None = None) -> bool: + from litellm._catalog import _read + + return _read.is_priced(model, provider) + + def reload_source(url: str) -> Source: if not url or is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": return Bundled() diff --git a/litellm/_catalog/_read.py b/litellm/_catalog/_read.py index a554e7f0ab4..7268a44a05d 100644 --- a/litellm/_catalog/_read.py +++ b/litellm/_catalog/_read.py @@ -7,7 +7,7 @@ from functools import wraps from itertools import chain from typing import TYPE_CHECKING, Final, Literal, ParamSpec, TypeVar, overload -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from litellm._catalog import _install from litellm.rust_bridge import _native @@ -154,6 +154,27 @@ def model_info(key: str) -> dict[str, object]: return {name: raw[name] for name in result.fields} +def is_priced(model: str, provider: str | None = None) -> bool: + resolved: Final = resolve(model, provider) + if not isinstance(resolved, _native.ResolvedModel): + return False + try: + row: Final = _ROW.validate_python( + dict[str, object].get( + _install._live_rows(), # pyright: ignore[reportPrivateUsage] # presence uses the selected live row + resolved.key, + ) + ) + except ValidationError: + return False + contextual_row: Final = row if "litellm_provider" in row else {"litellm_provider": provider or "custom", **row} + match _native.has_token_prices(contextual_row, policy="sdk_tokens"): + case bool() as present: + return present + case _native.RowErrors(): + return False + + def supports(fields: Mapping[str, object], capability: str) -> bool | None: value: Final = fields.get(capability) return None if value is None else value is True diff --git a/litellm/_cost/__init__.py b/litellm/_cost/__init__.py new file mode 100644 index 00000000000..822616c9478 --- /dev/null +++ b/litellm/_cost/__init__.py @@ -0,0 +1,23 @@ +from litellm._cost._price import ( + billing_context, + cache_read_meter, + cache_write_quantities, + evaluate, + price, + request_model_info, + request_prices, + require_priced, + token_usage, +) + +__all__ = ( + "billing_context", + "cache_read_meter", + "cache_write_quantities", + "evaluate", + "price", + "request_model_info", + "request_prices", + "require_priced", + "token_usage", +) diff --git a/litellm/_cost/_price.py b/litellm/_cost/_price.py new file mode 100644 index 00000000000..ccbfbe7e9e1 --- /dev/null +++ b/litellm/_cost/_price.py @@ -0,0 +1,252 @@ +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from typing import ( + Final, + Literal, + cast, # noqa: TID251 # validated request prices extend the read-only public ModelInfo shape +) + +from pydantic import TypeAdapter + +from litellm.exceptions import ModelNotMappedError +from litellm.rust_bridge import _native +from litellm.types.utils import CacheCreationTokenDetails, CustomPricingLiteLLMParams, ModelInfo, Usage + +_REQUEST_ROW: Final = TypeAdapter(dict[str, object]) +_REQUEST_PRICE_KEYS: Final = frozenset(CustomPricingLiteLLMParams.model_fields) + + +def _request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: + info: Final = kwargs.get("model_info") + if not isinstance(info, Mapping): + return {} + fields: Final = _REQUEST_ROW.validate_python(info) + return { + key: value + for key, value in fields.items() + if key in ("mode", "supports_prompt_caching", "max_tokens") and value is not None + } + + +def request_prices(kwargs: Mapping[str, object]) -> Mapping[str, object] | None: + has_tokens: Final = ( + kwargs.get("input_cost_per_token") is not None and kwargs.get("output_cost_per_token") is not None + ) + has_seconds: Final = any( + kwargs.get(key) is not None for key in ("cost_per_second", "input_cost_per_second", "output_cost_per_second") + ) + if not has_tokens and not has_seconds: + return None + prices: Final = CustomPricingLiteLLMParams.model_validate( + {key: value for key, value in kwargs.items() if key in _REQUEST_PRICE_KEYS and value is not None} + ) + return { + **_request_metadata(kwargs), + **_REQUEST_ROW.validate_python(prices.model_dump(exclude_none=True, exclude_unset=True)), + } + + +def request_model_info(model: str, provider: str | None, overrides: Mapping[str, object]) -> ModelInfo: + from litellm.utils import get_model_info + + def live_info() -> ModelInfo: + try: + return get_model_info(model=model, custom_llm_provider=provider) + except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unknown custom model + return { + "key": model, + "litellm_provider": provider or "custom", + "mode": "chat", + "input_cost_per_token": None, + "output_cost_per_token": None, + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "supported_openai_params": None, + } + + return cast( # cast-ok: validated pricing keys preserve extra request meters + ModelInfo, {**live_info(), **overrides} + ) + + +def price( + row: Mapping[str, object], + usage: _native.UsageInput, + when: _native.WhenInput, + *, + factors: Sequence[float] = (), + cache_read_default_ratio: float | None = None, + directional_token_threshold: int | None = None, +) -> _native.Priced | _native.Unpriced | _native.RowErrors: + if not factors: + return _native.price( + dict(row), + usage, + when, + policy="sdk_tokens", + cache_read_default_ratio=cache_read_default_ratio, + directional_token_threshold=directional_token_threshold, + ) + result: Final = evaluate( + row, + usage, + when, + factors=factors, + cache_read_default_ratio=cache_read_default_ratio, + directional_token_threshold=directional_token_threshold, + ) + return result if isinstance(result, _native.RowErrors) else result.pricing + + +def evaluate( + row: Mapping[str, object], + usage: _native.UsageInput, + when: _native.WhenInput, + *, + project: Sequence[str] = (), + factors: Sequence[float] = (), + cache_read_default_ratio: float | None = None, + directional_token_threshold: int | None = None, +) -> _native.PriceEvaluation | _native.RowErrors: + return _native.price_with_rates( + dict(row), + usage, + when, + policy="sdk_tokens", + project=project, + factors=factors, + cache_read_default_ratio=cache_read_default_ratio, + directional_token_threshold=directional_token_threshold, + ) + + +def require_priced(result: _native.Priced | _native.Unpriced | _native.RowErrors, model: str) -> _native.Priced: + if isinstance(result, _native.Priced): + return result + detail: Final = ( + result.missing + if isinstance(result, _native.Unpriced) + else tuple(f"{error.field}: {error.message}" for error in result.errors) + ) + raise ModelNotMappedError(f"No valid pricing for model={model}: {detail}") + + +def billing_context( + current_time: datetime, + service_tier: str | None = None, + channel: Literal["online", "batch", "decisions"] = "online", +) -> _native.WhenInput: + normalized: Final = service_tier.strip().lower() if service_tier else "standard" + tier: Final = ( + "priority" + if normalized == "fast" + else normalized + if normalized in ("priority", "flex", "ultrafast", "balanced") + else "standard" + ) + utc_time: Final = current_time if current_time.tzinfo is not None else current_time.replace(tzinfo=timezone.utc) + return _native.WhenInput(int(utc_time.timestamp()), tier=tier, channel=channel) + + +def cache_read_meter(usage: Usage) -> Literal["cache_read_input_token_cost", "explicit_cache_read_input_token_cost"]: + return ( + "explicit_cache_read_input_token_cost" + if getattr(usage.prompt_tokens_details, "cache_type", None) == "ephemeral" + else "cache_read_input_token_cost" + ) + + +def cache_write_quantities(cache_creation: int, ttl: CacheCreationTokenDetails | None) -> Mapping[str, int]: + short_writes: Final = ((ttl.ephemeral_5m_input_tokens or 0) + (ttl.ephemeral_30m_input_tokens or 0)) if ttl else 0 + writes: Final = { + "cache_creation_input_token_cost_above_1hr": (ttl.ephemeral_1h_input_tokens or 0) if ttl else 0, + "cache_creation_input_token_cost_above_12hr": (ttl.ephemeral_12h_input_tokens or 0) if ttl else 0, + } + return { + "cache_creation_input_token_cost": max(cache_creation - short_writes - sum(writes.values()), 0) + short_writes, + **writes, + } + + +def _prompt_quantities(usage: Usage, row: Mapping[str, object]) -> Mapping[str, int]: + from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details + + if usage.prompt_tokens_details is None: + return {"input_cost_per_token": usage.prompt_tokens} + details: Final = parse_prompt_tokens_details(usage) + cache_hit: Final = details["cache_hit_tokens"] + cache_creation: Final = details["cache_creation_tokens"] + total_details: Final = sum( + ( + details["text_tokens"], + cache_hit, + cache_creation, + details["audio_tokens"], + details["image_tokens"], + details["video_tokens"], + ) + ) + overlaps: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens + uncached: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0) + audio: Final = min(details["audio_tokens"], uncached) if overlaps else details["audio_tokens"] + image: Final = min(details["image_tokens"], uncached - audio) if overlaps else details["image_tokens"] + video: Final = min(details["video_tokens"], uncached - audio - image) if overlaps else details["video_tokens"] + remainder: Final = max(uncached - audio - image - video, 0) + text: Final = ( + remainder + if overlaps or (details["text_tokens"] == 0 and details["image_count"] == 0) + else details["text_tokens"] + ) + return { + "input_cost_per_token": 0 + if details["character_count"] and row.get("input_cost_per_character") is not None + else text, + "input_cost_per_audio_token": 0 + if details["audio_length_seconds"] and row.get("input_cost_per_audio_per_second") is not None + else audio, + "input_cost_per_image_token": 0 + if details["image_count"] and row.get("input_cost_per_image") is not None + else image, + "input_cost_per_video_token": 0 + if details["video_length_seconds"] and row.get("input_cost_per_video_per_second") is not None + else video, + cache_read_meter(usage): cache_hit - details["cache_hit_audio_tokens"], + "cache_read_input_audio_token_cost": details["cache_hit_audio_tokens"], + **cache_write_quantities(cache_creation, details["cache_creation_token_details"]), + } + + +def _completion_quantities(usage: Usage, *, batch: bool = False) -> Mapping[str, int]: + from litellm.litellm_core_utils.llm_cost_calc.utils import parse_completion_tokens_details + + if usage.completion_tokens_details is None: + return {"output_cost_per_token": usage.completion_tokens} + details: Final = parse_completion_tokens_details(usage) + image_tokens: Final = min(details["image_tokens"], usage.completion_tokens) if batch else details["image_tokens"] + modalities: Final = { + "output_cost_per_audio_token": details["audio_tokens"], + "output_cost_per_reasoning_token": details["reasoning_tokens"], + "output_cost_per_image_token": image_tokens, + "output_cost_per_video_token": details["video_tokens"], + } + text: Final = details["text_tokens"] or max(usage.completion_tokens - sum(modalities.values()), 0) + return {"output_cost_per_token": text, **modalities} + + +def token_usage( + usage: Usage, + row: Mapping[str, object], + *, + inclusive_threshold: bool = False, + batch: bool = False, + direction: Literal["input", "output"] | None = None, +) -> _native.UsageInput: + return _native.UsageInput( + { + **(_prompt_quantities(usage, row) if direction != "output" else {}), + **(_completion_quantities(usage, batch=batch) if direction != "input" else {}), + }, + prompt_tokens=usage.prompt_tokens, + inclusive_threshold=inclusive_threshold, + ) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 2440e80aaab..8a2895bb459 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -16,6 +16,8 @@ from typing_extensions import ReadOnly, TypedDict, assert_never import litellm import litellm._logging from litellm import verbose_logger +from litellm._cost import request_model_info +from litellm._internal_context import current_billing_time from litellm.constants import ( DEFAULT_MAX_LRU_CACHE_SIZE, DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND, @@ -37,17 +39,11 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( _SERVICE_TIER_TO_COST_KEY_SUFFIX, BilledTokenRates, CostCalculatorUtils, - calculate_cache_writing_cost, - calculate_cost_component, generic_cost_per_character, generic_cost_per_token, - get_batch_cost_rates, - get_billable_input_tokens, get_regional_uplift_multiplier, - get_service_tier_cost_key, get_token_type_cost_breakdown, get_vertex_regional_endpoint_uplift, - parse_prompt_tokens_details, select_cost_metric_for_model, ) from litellm.llms.anthropic.cost_calculation import ( @@ -239,30 +235,26 @@ def _cost_per_token_custom_pricing_helper( return None if custom_cost_per_token is not None: - input_cost_per_token: Final = custom_cost_per_token["input_cost_per_token"] - output_cost_per_token: Final = custom_cost_per_token["output_cost_per_token"] + from litellm._cost import billing_context, price, require_priced + from litellm.rust_bridge import _native - cache_read_input_token_cost: Final = custom_cost_per_token.get( - "cache_read_input_token_cost", - input_cost_per_token, + priced: Final = require_priced( + price( + {"litellm_provider": "custom", **custom_cost_per_token}, + _native.UsageInput( + { + "input_cost_per_token": int(max(prompt_tokens - cached_tokens - cache_creation_tokens, 0)), + "cache_read_input_token_cost": int(cached_tokens), + "cache_creation_input_token_cost": int(cache_creation_tokens), + "output_cost_per_token": int(completion_tokens), + }, + prompt_tokens=int(prompt_tokens), + ), + billing_context(current_billing_time()), + ), + "custom pricing", ) - cache_creation_input_token_cost: Final = custom_cost_per_token.get( - "cache_creation_input_token_cost", - input_cost_per_token, - ) - - regular_prompt_tokens: Final = max( - prompt_tokens - cached_tokens - cache_creation_tokens, - 0, - ) - - input_cost: Final = ( - regular_prompt_tokens * input_cost_per_token - + cached_tokens * cache_read_input_token_cost - + cache_creation_tokens * cache_creation_input_token_cost - ) - output_cost = completion_tokens * output_cost_per_token - return input_cost, output_cost + return priced.input, priced.output elif custom_cost_per_second is not None: output_cost = custom_cost_per_second * (response_time_ms or 0.0) / 1000 return 0, output_cost @@ -369,9 +361,9 @@ def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool: return mode is None or mode in _WALL_CLOCK_PRICED_MODES -def _model_info_for_pricing(model: str, custom_llm_provider: str | None) -> ModelInfoBase | None: +def _model_info_for_pricing(model: str, custom_llm_provider: str | None) -> ModelInfo | None: try: - return cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model return None @@ -807,18 +799,16 @@ def cost_per_token( model_info: Final = ( pricing_model_info if pricing_model_info is not None - else cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + else litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + ) + return generic_cost_per_token( + model=model, + usage=usage_block, + custom_llm_provider=custom_llm_provider, + service_tier=service_tier, + data_residency=data_residency, + model_info=model_info, ) - if _has_token_or_tiered_pricing(model_info): - return generic_cost_per_token( - model=model, - usage=usage_block, - custom_llm_provider=custom_llm_provider, - service_tier=service_tier, - data_residency=data_residency, - ) - verbose_logger.debug("No per-token, tiered, or per-second pricing for model=%s; cost is 0", model) - return 0.0, 0.0 def get_replicate_completion_pricing(completion_response: dict, total_time=0.0): @@ -909,7 +899,12 @@ def _bedrock_pricing_info( ) profile: Final = _cost_map_model_info(invoked_model_id, "bedrock") if invoked_model_id else None if profile is None or not _declares_rates(profile): - return _cost_map_model_info(registration[0], None) if registration is not None else None + if registration is None: + return None + source: Final = _raw_cost_map_entry(registration[0]) + return _cost_map_model_info( + registration[0], "bedrock" if source is not None and "litellm_provider" not in source else None + ) if registration is None or registration[0] == profile["key"]: return profile return model_info_with_pricing_overrides(profile, registration[1]) @@ -1512,6 +1507,35 @@ def _response_time_ms_for_cost( return 0.0 +def _request_pricing_cost( + *, + model: str, + provider: str | None, + prices: Mapping[str, object] | None, + usage: Usage, + response_time_ms: float | None, + audio_seconds: float, + service_tier: str | None, + data_residency: str | None, + vertex_location: str | None, +) -> tuple[ModelInfo | None, tuple[float, float] | None]: + if prices is None: + return None, None + info: Final = request_model_info(model, provider, prices) + time_cost: Final = _per_second_pricing_cost(model, info, response_time_ms, audio_seconds) + if time_cost is not None: + return info, time_cost + return info, generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=provider or "custom", + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + model_info=info, + ) + + def completion_cost( completion_response: object | None = None, model: str | None = None, @@ -2032,36 +2056,54 @@ def completion_cost( if litellm_logging_obj is not None: request_model_for_cost = litellm_logging_obj.model - ( - prompt_tokens_cost_usd_dollar, - completion_tokens_cost_usd_dollar, - ) = cost_per_token( + request_info, request_cost = _request_pricing_cost( model=model, - prompt_tokens=prompt_tokens or 0, - completion_tokens=completion_tokens or 0, - custom_llm_provider=_pricing_provider_for_cost_candidate( - candidate_provider=candidate_pricing_provider, - request_provider=request_pricing_provider, - current_provider=custom_llm_provider, - ), + provider=custom_llm_provider, + prices=litellm_logging_obj.request_pricing + if custom_pricing and litellm_logging_obj is not None and custom_cost_per_token is None + else None, + usage=cost_per_token_usage_object + or Usage(prompt_tokens=prompt_tokens or 0, completion_tokens=completion_tokens or 0), response_time_ms=response_time_ms, - region_name=None if explicit_pricing else region_name, - custom_cost_per_second=custom_cost_per_second, - custom_cost_per_token=custom_cost_per_token, - prompt_characters=prompt_characters, - completion_characters=completion_characters, - cache_creation_input_tokens=cache_creation_input_tokens, - cache_read_input_tokens=cache_read_input_tokens, - usage_object=cost_per_token_usage_object, - call_type=call_type, - audio_transcription_file_duration=audio_transcription_file_duration, - rerank_billed_units=rerank_billed_units, + audio_seconds=audio_transcription_file_duration or 0.0, service_tier=service_tier, data_residency=data_residency, vertex_location=vertex_location, - response=completion_response, - custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id), - bedrock_model_info=bedrock_model_info, + ) + ( + prompt_tokens_cost_usd_dollar, + completion_tokens_cost_usd_dollar, + ) = ( + request_cost + if request_cost is not None + else cost_per_token( + model=model, + prompt_tokens=prompt_tokens or 0, + completion_tokens=completion_tokens or 0, + custom_llm_provider=_pricing_provider_for_cost_candidate( + candidate_provider=candidate_pricing_provider, + request_provider=request_pricing_provider, + current_provider=custom_llm_provider, + ), + response_time_ms=response_time_ms, + region_name=None if explicit_pricing else region_name, + custom_cost_per_second=custom_cost_per_second, + custom_cost_per_token=custom_cost_per_token, + prompt_characters=prompt_characters, + completion_characters=completion_characters, + cache_creation_input_tokens=cache_creation_input_tokens, + cache_read_input_tokens=cache_read_input_tokens, + usage_object=cost_per_token_usage_object, + call_type=call_type, + audio_transcription_file_duration=audio_transcription_file_duration, + rerank_billed_units=rerank_billed_units, + service_tier=service_tier, + data_residency=data_residency, + vertex_location=vertex_location, + response=completion_response, + custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id), + bedrock_model_info=bedrock_model_info, + ) ) # Get additional costs from provider (e.g., routing fees, infrastructure costs) @@ -2153,7 +2195,7 @@ def completion_cost( data_residency=data_residency, vertex_location=vertex_location, custom_cost_per_token=custom_cost_per_token, - model_info=bedrock_model_info, + model_info=request_info if request_info is not None else bedrock_model_info, ) _reasoning_cost = _token_type_breakdown.reasoning_cost _cache_read_cost = _token_type_breakdown.cache_read_cost @@ -2889,32 +2931,26 @@ def default_video_cost_calculator( return 0.0 -def _batch_rate( - model_info: ModelInfo, - key: Literal[ - "input_cost_per_audio_token_batches", - "cache_read_input_audio_token_cost_batches", - "input_cost_per_image_token_batches", - "input_cost_per_video_token_batches", - ], - fallback: float, -) -> float: - rate: Final = model_info.get(key) - return fallback if rate is None else rate - - -def _batch_or_half(batch_rate: float | None, standard_rate: float | None, default: float = 0.0) -> float: - if batch_rate is not None: - return batch_rate - if standard_rate is not None: - return standard_rate / 2 - return default - - -def _completion_image_tokens(usage: Usage) -> int: - details: Final = usage.completion_tokens_details - image_tokens: Final = (details.image_tokens or 0) if details is not None else 0 - return min(image_tokens, usage.completion_tokens) +def _batch_model_info(model: str, provider: str | None, model_info: ModelInfo | None) -> ModelInfo | None: + if model_info is not None and any( + model_info.get(key) is not None + for key in ( + "input_cost_per_token_batches", + "input_cost_per_token", + "output_cost_per_token_batches", + "output_cost_per_token", + ) + ): + return model_info + try: + global_info: Final = litellm.get_model_info(model=model, custom_llm_provider=provider) + except Exception: + return model_info + if model_info is None: + return global_info + return cast( # cast-ok: both public ModelInfo rows keep their validated field types on merge + ModelInfo, {**global_info, **{key: value for key, value in model_info.items() if value is not None}} + ) def batch_cost_calculator( @@ -2926,143 +2962,28 @@ def batch_cost_calculator( *, vertex_location: str | None = None, ) -> tuple[float, float]: - """ - Calculate the cost of a batch job. + from litellm._cost import billing_context, price, require_priced, token_usage + from litellm.litellm_core_utils.llm_cost_calc.utils import uses_inclusive_token_thresholds - Args: - model_info: Optional deployment-level model info containing custom - batch pricing (e.g. input_cost_per_token_batches). When provided, - skips the global litellm.get_model_info() lookup so that - deployment-specific pricing is used. - """ - - _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) - - verbose_logger.debug( - "Calculating batch cost per token. model=%s, custom_llm_provider=%s", - model, - custom_llm_provider, - ) - - if model_info is None: - try: - model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) - except Exception: - model_info = None - elif not any( - model_info.get(k) is not None - for k in ( - "input_cost_per_token_batches", - "input_cost_per_token", - "output_cost_per_token_batches", - "output_cost_per_token", - ) - ): - deployment_info: Final = model_info - try: - global_info: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) - if global_info: - model_info = { - **global_info, - **{key: value for key, value in deployment_info.items() if value is not None}, - } - except Exception: - pass - - if not model_info: - return 0.0, 0.0 - - batch_rates: Final = get_batch_cost_rates(model_info, usage, custom_llm_provider) - input_cost_per_token: Final = model_info.get("input_cost_per_token") - output_cost_per_token: Final = model_info.get("output_cost_per_token") - total_prompt_cost = 0.0 - total_completion_cost = 0.0 - if batch_rates.input is not None: - total_prompt_cost = _batch_prompt_cost( - usage, - model_info, - batch_rates.input, - batch_rates.cache_read, - batch_rates.cache_creation, - batch_rates.cache_creation_rates_by_ttl, - ) - elif input_cost_per_token: - details: Final = parse_prompt_tokens_details(usage) - cache_read_tokens: Final = details["cache_hit_tokens"] - cache_creation_tokens: Final = details["cache_creation_tokens"] - - # Subtract cached tokens from prompt_tokens before calculating cost - # Fixes issue where cached tokens are being charged again - base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens - total_prompt_cost = ( - base_input_tokens * (input_cost_per_token) / 2 - ) # batch cost is usually half of the regular token cost - - # Add cache read cost if applicable - cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None) - total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2 - - cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token - total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2 - text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token) - image_rate: Final = _batch_or_half( - model_info.get("output_cost_per_image_token_batches"), - model_info.get("output_cost_per_image_token"), - default=text_rate, - ) - image_tokens: Final = _completion_image_tokens(usage) - text_tokens: Final = usage.completion_tokens - image_tokens - total_completion_cost = text_tokens * text_rate + image_tokens * image_rate - - uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) - if uplift != 1.0: - total_prompt_cost *= uplift - total_completion_cost *= uplift - - vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) - return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift - - -def _batch_prompt_cost( - usage: Usage, - model_info: ModelInfo, - input_rate: float, - cache_read_rate: float | None, - cache_creation_rate: float | None, - cache_creation_rates_by_ttl: Mapping[str, float], -) -> float: - details: Final = parse_prompt_tokens_details(usage) - cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 - cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 - bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl) - written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0 - audio_tokens, image_tokens, video_tokens = ( - details["audio_tokens"], - details["image_tokens"], - details["video_tokens"], - ) - text_tokens: Final = max( - (usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens - cached_tokens - written_tokens, 0 - ) - return ( - text_tokens * input_rate - + audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate) - + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate) - + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) - + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) - + cached_audio_tokens - * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0) - + ( - calculate_cache_writing_cost( - written_tokens, - details["cache_creation_token_details"], - input_rate if cache_creation_rate is None else cache_creation_rate, - cache_creation_rates_by_ttl, + _, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + resolved: Final = _batch_model_info(model, provider, model_info) + row: Final = dict(resolved or {}) + pricing_row: Final = row if "litellm_provider" in row else {"litellm_provider": provider or "custom", **row} + priced: Final = require_priced( + price( + pricing_row, + token_usage(usage, pricing_row, inclusive_threshold=uses_inclusive_token_thresholds(provider), batch=True), + billing_context(current_billing_time(), channel="batch"), + factors=( + get_regional_uplift_multiplier(resolved, data_residency), + get_vertex_regional_endpoint_uplift(resolved, vertex_location), ) - if bills_cache_writes - else 0.0 - ) + if resolved is not None + else (), + ), + model, ) + return priced.input, priced.output _ATTRIBUTE_VALUE: Final = TypeAdapter(object) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 9e0461e951d..54095233184 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -626,6 +626,7 @@ class Logging(LiteLLMLoggingBaseClass): slack_app custom_pricing: bool = False custom_cost_per_token: CostPerToken | None = None + request_pricing: Mapping[str, object] | None = None custom_cost_per_second: float | None = None replaced_global_callback_keys: frozenset[str] = frozenset() stream_options = None @@ -804,6 +805,11 @@ class Logging(LiteLLMLoggingBaseClass): self.litellm_params = litellm_params + if kwargs is not None: + from litellm._cost import request_prices + + self.request_pricing = request_prices(kwargs) + # Initialize cost breakdown field self.cost_breakdown: CostBreakdown | None = None self.billed_token_rates: BilledTokenRates | None = None diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 3b49f239afe..fe19191e11e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -3,22 +3,19 @@ import re from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field, replace -from datetime import datetime, timezone, tzinfo +from dataclasses import dataclass +from datetime import datetime +from math import prod from types import MappingProxyType from typing import Final, Literal, TypedDict, cast -from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from typing_extensions import ReadOnly import litellm from litellm._internal_context import current_billing_time from litellm._logging import verbose_logger -from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import ( - select_tier_for_input, - tier_rate, -) -from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate +from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO +from litellm.rust_bridge import _native from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, @@ -97,26 +94,14 @@ SERVICE_TIER_COST_KEY_SUFFIXES: Final[tuple[str, ...]] = tuple( ) _INCLUSIVE_THRESHOLD_PROVIDERS: Final = frozenset({"xai"}) -_BATCH_KEY_SUFFIX: Final = "_batches" -_BATCH_RATE_PREFIXES: Final = ( - "input_cost_per_token", - "output_cost_per_token", - "cache_read_input_token_cost", - "cache_creation_input_token_cost", -) -_BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values()) -_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_TIER_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$") -_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX) -def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool: +def uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool: return custom_llm_provider in _INCLUSIVE_THRESHOLD_PROVIDERS -def apply_provider_cache_read_default(model_info: ModelInfo, custom_llm_provider: str | None) -> ModelInfo: - if custom_llm_provider == "fireworks_ai": - return with_default_cache_read_rate(model_info) - return model_info +def cache_read_default_ratio(custom_llm_provider: str | None) -> float | None: + return FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO if custom_llm_provider == "fireworks_ai" else None def _get_token_detail_value(details: object, key: str) -> int | None: @@ -297,601 +282,6 @@ def get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str: _get_service_tier_cost_key = get_service_tier_cost_key -def _parse_token_threshold(threshold: str) -> float: - return float(threshold.replace("k", "")) * (1000 if "k" in threshold else 1) - - -def _parse_above_token_threshold(key: str) -> float: - return _parse_token_threshold(key.rsplit("_above_", 1)[1].split("_tokens")[0]) - - -def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool: - return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold) - - -@dataclass(frozen=True, slots=True) -class BatchCostRates: - input: float | None - output: float | None - cache_read: float | None - cache_creation: float | None - cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({})) - - -def _batch_rate(model_info: ModelInfo, key: str) -> float | None: - value: Final = model_info.get(key) - if isinstance(value, (int, float)): - return float(value) - if not isinstance(value, str): - return None - try: - return float(value) - except ValueError: - return None - - -def _batch_tier_rate(model_info: ModelInfo, tier_key: str, flat_key: str) -> float | None: - tier_rate: Final = _batch_rate(model_info, tier_key) - return _batch_rate(model_info, flat_key) if tier_rate is None else tier_rate - - -def _batch_tier_thresholds(model_info: ModelInfo, prefix: str) -> frozenset[str]: - return frozenset( - tier.group(2) - for key, value in model_info.items() - if value is not None and (tier := _BATCH_TIER_KEY.match(key)) is not None and tier.group(1) == prefix - ) - - -def _crossed_batch_tier(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> str | None: - return next( - ( - threshold - for threshold in sorted( - _batch_tier_thresholds(model_info, prefix), key=_parse_token_threshold, reverse=True - ) - if _prompt_exceeds_threshold(usage.prompt_tokens, _parse_token_threshold(threshold), inclusive) - ), - None, - ) - - -def _batch_rate_for_prefix(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> float | None: - flat_key: Final = f"{prefix}{_BATCH_KEY_SUFFIX}" - threshold: Final = _crossed_batch_tier(model_info, prefix, usage, inclusive) - if threshold is None: - return _batch_rate(model_info, flat_key) - return _batch_tier_rate(model_info, f"{prefix}_above_{threshold}_tokens{_BATCH_KEY_SUFFIX}", flat_key) - - -def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provider: str | None) -> BatchCostRates: - inclusive: Final = _uses_inclusive_token_thresholds(custom_llm_provider) - return BatchCostRates( - input=_batch_rate_for_prefix(model_info, "input_cost_per_token", usage, inclusive), - output=_batch_rate_for_prefix(model_info, "output_cost_per_token", usage, inclusive), - cache_read=_batch_rate_for_prefix(model_info, "cache_read_input_token_cost", usage, inclusive), - cache_creation=_batch_rate_for_prefix(model_info, "cache_creation_input_token_cost", usage, inclusive), - cache_creation_rates_by_ttl=MappingProxyType( - { - ttl: rate - for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items() - if (rate := _batch_rate_for_prefix(model_info, key, usage, inclusive)) is not None - } - ), - ) - - -@dataclass(frozen=True, slots=True) -class TokenRates: - input_rate: float - output_rate: float - cache_read_rate: float - cache_creation_rate: float - reasoning_rate: float | None - cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({})) - - @property - def billed_reasoning_rate(self) -> float: - return self.output_rate if self.reasoning_rate is None else self.reasoning_rate - - -def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> Mapping[str, object] | None: - tiered_pricing: Final = model_info.get("tiered_pricing") - if not isinstance(tiered_pricing, list) or not tiered_pricing: - return None - - tier: Final = select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=usage.prompt_tokens) - if tier is None or "input_cost_per_token" not in tier: - return None - return tier - - -def _get_tiered_reasoning_rate(model_info: ModelInfo, usage: Usage) -> float | None: - tier: Final = _select_priced_tier(model_info=model_info, usage=usage) - if tier is None: - return None - if "output_cost_per_reasoning_token" not in tier and "output_cost_per_token" not in tier: - return None - return tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token") - - -def _get_tiered_base_costs(model_info: ModelInfo, usage: Usage) -> tuple[TokenRates, Mapping[str, object]] | None: - """ - Resolve the base rates from a model's ``tiered_pricing`` table, if it has one. - - Tiered pricing is all-or-nothing: one tier is picked from the request's input tokens - and every token of the request is billed at that tier's rate. Rates the tier does not - declare fall back to the tier's input rate, so a request never mixes tiers. - - An output rate is the exception: a tier table that spells out only input rates would - otherwise serve every completion for free, so the model's own output rate stands in. - """ - tier: Final = _select_priced_tier(model_info=model_info, usage=usage) - if tier is None: - return None - - completion_cost: Final = ( - tier_rate(tier, "output_cost_per_token") - if "output_cost_per_token" in tier - else get_cost_per_unit(model_info, "output_cost_per_token") or 0.0 - ) - cache_creation_rate: Final = tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token") - rates: Final = TokenRates( - input_rate=tier_rate(tier, "input_cost_per_token"), - output_rate=completion_cost, - cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"), - cache_creation_rate=cache_creation_rate, - reasoning_rate=None, - ) - return rates, tier - - -def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_time: datetime | None = None) -> bool: - """Return True if current_time (UTC, defaulting to now) falls inside any off-peak window. - - off_peak_hours_utc is a "HH:MM-HH:MM" string in UTC, or a list of such strings for providers - with multiple daily windows (e.g. ["16:30-00:30", "04:00-06:00"]). A window may wrap past - midnight, and a window whose start equals its end covers the whole day. The start is - inclusive and the end is exclusive; malformed windows are ignored. - - An aware current_time is converted to UTC. A naive one is taken to already be UTC rather - than being localised, so callers must pass datetime.now(timezone.utc), never datetime.now(), - or every window shifts by the host's offset. - """ - reference: Final = current_time if current_time is not None else current_billing_time() - now: Final = (reference.astimezone(timezone.utc) if reference.tzinfo is not None else reference).time() - windows: Final = (off_peak_hours_utc,) if isinstance(off_peak_hours_utc, str) else off_peak_hours_utc - for window in windows: - try: - start_str, end_str = window.split("-") - start = datetime.strptime(start_str.strip(), "%H:%M").replace(tzinfo=timezone.utc).time() - end = datetime.strptime(end_str.strip(), "%H:%M").replace(tzinfo=timezone.utc).time() - except (ValueError, AttributeError): - continue - if start < end: - if start <= now < end: - return True - elif now >= start or now < end: - return True - return False - - -_WEEKDAY_NUMBERS: Final = MappingProxyType( - { - "mon": 1, - "monday": 1, - "tue": 2, - "tues": 2, - "tuesday": 2, - "wed": 3, - "wednesday": 3, - "thu": 4, - "thur": 4, - "thurs": 4, - "thursday": 4, - "fri": 5, - "friday": 5, - "sat": 6, - "saturday": 6, - "sun": 7, - "sunday": 7, - } -) - - -def _normalize_weekday(value: object) -> int | None: - if isinstance(value, bool): - return None - if isinstance(value, int): - return value if 1 <= value <= 7 else None - if isinstance(value, str): - return _WEEKDAY_NUMBERS.get(value.strip().lower()) - return None - - -def _weekday_calendar(weekday_timezone: object) -> tzinfo: - if isinstance(weekday_timezone, str) and weekday_timezone.strip(): - try: - return ZoneInfo(weekday_timezone.strip()) - except (ValueError, ZoneInfoNotFoundError): - return timezone.utc - return timezone.utc - - -def _matches_weekdays(reference_utc: datetime, weekdays: object, weekday_timezone: object) -> bool: - """Return True when reference_utc falls on one of the rule's weekdays, read on the calendar - named by weekday_timezone (default UTC). An absent weekdays means every day. The calendar - matters even when UTC and vendor-local weekdays agree at every currently priced hour: a - window past 16:00 UTC is where an Asia/Shanghai weekday diverges from the UTC one. - """ - if weekdays is None: - return True - if isinstance(weekdays, str) or not isinstance(weekdays, Sequence): - return False - allowed: Final = frozenset(day for day in map(_normalize_weekday, weekdays) if day is not None) - return reference_utc.astimezone(_weekday_calendar(weekday_timezone)).isoweekday() in allowed - - -def _as_window_strings(value: object) -> tuple[str, ...]: - if isinstance(value, str): - return (value,) - if isinstance(value, Sequence): - return tuple(entry for entry in value if isinstance(entry, str)) - return () - - -def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None = None) -> bool: - """Return True when current_time (UTC, defaulting to now) is off-peak under the block's - rules: the flat hours_utc windows, which apply every day, or any entry in windows, whose - hours apply only on its weekdays. - """ - reference: Final = current_time if current_time is not None else current_billing_time() - reference_utc: Final = ( - reference.astimezone(timezone.utc) if reference.tzinfo is not None else reference.replace(tzinfo=timezone.utc) - ) - flat_windows: Final = _as_window_strings(off_peak.get("hours_utc")) - if flat_windows and _is_within_off_peak_window(flat_windows, reference_utc): - return True - windows: Final = off_peak.get("windows") - if isinstance(windows, str) or not isinstance(windows, Sequence): - return False - weekday_timezone: Final = off_peak.get("weekday_timezone") - for rule in windows: - if not isinstance(rule, Mapping): - continue - rule_windows = _as_window_strings(rule.get("hours_utc")) - if not rule_windows: - continue - if not _matches_weekdays(reference_utc, rule.get("weekdays"), weekday_timezone): - continue - if _is_within_off_peak_window(rule_windows, reference_utc): - return True - return False - - -def resolve_explicit_cache_read_cost_rate( - model_info: ModelInfo, - usage: Usage, - current_time: datetime | None, - service_tier: str | None = None, -) -> float | None: - cache_read_cost_key: Final = "explicit_cache_read_input_token_cost" - service_cache_read_cost_key: Final = _get_service_tier_cost_key(cache_read_cost_key, service_tier) - if ( - model_info.get(service_cache_read_cost_key) is None - and model_info.get(cache_read_cost_key) is None - and model_info.get("tiered_pricing") is None - and model_info.get("off_peak_pricing") is None - ): - return None - - prompt_tokens_details: Final = usage.prompt_tokens_details - cache_type: Final = ( - getattr(prompt_tokens_details, "cache_type", None) if prompt_tokens_details is not None else None - ) - if parse_prompt_tokens_details(usage)["cache_hit_tokens"] <= 0 or cache_type != "ephemeral": - return None - - tier: Final = _select_priced_tier(model_info=model_info, usage=usage) - off_peak: Final = _open_off_peak_block(model_info, current_time) - model_cache_read_cost_rate: Final = _get_cost_per_unit( - model_info, - service_cache_read_cost_key, - default_value=None, - ) - tier_has_explicit_rate: Final = tier is not None and cache_read_cost_key in tier - off_peak_has_explicit_rate: Final = ( - off_peak is not None and _parse_off_peak_rate(off_peak.get(cache_read_cost_key)) is not None - ) - if model_cache_read_cost_rate is None and not tier_has_explicit_rate and not off_peak_has_explicit_rate: - return None - - cache_read_cost_rate: Final = ( - tier_rate(tier, cache_read_cost_key) - if tier is not None and tier_has_explicit_rate - else tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token") - if tier is not None - else model_cache_read_cost_rate - if model_cache_read_cost_rate is not None - else _get_cost_per_unit( - model_info, - _get_service_tier_cost_key("cache_read_input_token_cost", service_tier), - default_value=_get_cost_per_unit(model_info, "input_cost_per_token") or 0.0, - ) - or 0.0 - ) - - if off_peak is None: - return cache_read_cost_rate - mode_specific_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get(cache_read_cost_key)) - generic_off_peak_rate: Final = _parse_off_peak_rate(off_peak.get("cache_read_input_token_cost")) - return ( - mode_specific_off_peak_rate - if mode_specific_off_peak_rate is not None - else generic_off_peak_rate - if generic_off_peak_rate is not None - else cache_read_cost_rate - ) - - -def _parse_off_peak_rate(value: object) -> float | None: - if isinstance(value, bool): - return None - if isinstance(value, (int, float)): - return float(value) - if isinstance(value, str): - try: - return float(value) - except ValueError: - return None - return None - - -def _off_peak_rate(off_peak: Mapping[str, object], key: str, standard_rate: float) -> float: - parsed: Final = _parse_off_peak_rate(off_peak.get(key)) - return standard_rate if parsed is None else parsed - - -def _open_off_peak_block(model_info: ModelInfo, current_time: datetime | None) -> Mapping[str, object] | None: - off_peak: Final = model_info.get("off_peak_pricing") - if not isinstance(off_peak, Mapping) or not _is_off_peak(off_peak, current_time): - return None - return off_peak - - -def apply_off_peak_pricing(model_info: ModelInfo, current_time: datetime | None, rates: TokenRates) -> TokenRates: - """Swap in off-peak per-token rates when the current UTC time is inside one of the model's - off_peak_pricing rules, the every-day hours_utc windows or a day-of-week-qualified entry in - windows. An off-peak rate replaces the rate that would otherwise apply rather than - discounting it, so a model that also has tiered or above-threshold pricing bills the flat - off-peak rate for the whole request while the window is open. Any rate left unset in - off_peak_pricing falls back to the standard rate, so a block without - output_cost_per_reasoning_token keeps the model's own reasoning rate, or its off-peak output - rate when reasoning has no dedicated rate at all. - """ - off_peak: Final = _open_off_peak_block(model_info, current_time) - if off_peak is None: - return rates - off_peak_reasoning_rate: Final = _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token")) - return replace( - rates, - input_rate=_off_peak_rate(off_peak, "input_cost_per_token", rates.input_rate), - output_rate=_off_peak_rate(off_peak, "output_cost_per_token", rates.output_rate), - cache_read_rate=_off_peak_rate(off_peak, "cache_read_input_token_cost", rates.cache_read_rate), - cache_creation_rate=_off_peak_rate(off_peak, "cache_creation_input_token_cost", rates.cache_creation_rate), - reasoning_rate=rates.reasoning_rate if off_peak_reasoning_rate is None else off_peak_reasoning_rate, - ) - - -def _apply_off_peak_to_base_costs( - model_info: ModelInfo, - current_time: datetime | None, - base_rates: TokenRates, - tier: Mapping[str, object], -) -> TokenRates: - """Apply off-peak rates to the base costs resolved from a pricing tier. off_peak_pricing has no - field for the long-TTL cache-creation rates, so a tier that declares one keeps it, and a tier - that does not bills that TTL at the off-peak-adjusted write rate. Reasoning is left to - _resolve_billed_reasoning_rate. - """ - rates: Final = apply_off_peak_pricing(model_info, current_time, base_rates) - return replace( - rates, - cache_creation_rates_by_ttl=MappingProxyType( - { - ttl: rates.cache_creation_rate if tier.get(key) is None else tier_rate(tier, key) - for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items() - } - ), - ) - - -def _first_present_key(model_info: ModelInfo, preferred_key: str, fallback_key: str) -> str: - return preferred_key if model_info.get(preferred_key) is not None else fallback_key - - -def _get_token_base_cost( - model_info: ModelInfo, - usage: Usage, - service_tier: str | None = None, - current_time: datetime | None = None, - *, - threshold_is_inclusive: bool = False, -) -> TokenRates: - """ - Return prompt cost, completion cost, and cache costs for a given model and usage. - - If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set, - then we use the corresponding threshold cost for all token types. - - `threshold_is_inclusive` switches that comparison to >=, for providers such as xAI - that bill the higher tier once the prompt reaches the threshold. - - An absent cache-creation or cache-read rate always resolves to the resolved input - rate, the way the tiered table and custom deployment pricing already do, since a - provider that publishes no cache price bills cached tokens as ordinary input. An - absent long-TTL write rate resolves to the cache-creation rate, off-peak included. An - explicit 0.0 stays a real price for all of them. - """ - tiered_base_costs: Final = _get_tiered_base_costs(model_info=model_info, usage=usage) - if tiered_base_costs is not None: - return _apply_off_peak_to_base_costs(model_info, current_time, *tiered_base_costs) - - # Get service tier aware cost keys - input_cost_key: Final = get_service_tier_cost_key("input_cost_per_token", service_tier) - output_cost_key: Final = get_service_tier_cost_key("output_cost_per_token", service_tier) - cache_creation_cost_key: Final = get_service_tier_cost_key("cache_creation_input_token_cost", service_tier) - cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", service_tier) - - prompt_base_cost = cast( # cast-ok: model pricing data is external - float, get_cost_per_unit(model_info, input_cost_key) - ) - completion_base_cost = cast( # cast-ok: model pricing data is external - float, get_cost_per_unit(model_info, output_cost_key) - ) - - # For image generation models that don't have output_cost_per_token, - # use output_cost_per_image_token as the base cost (all output tokens are image tokens) - if completion_base_cost == 0.0: - output_image_cost: Final = get_cost_per_unit(model_info, "output_cost_per_image_token", None) - if output_image_cost is not None: - completion_base_cost = output_image_cost - cache_creation_cost = get_cost_per_unit(model_info, cache_creation_cost_key, default_value=None) - cache_creation_keys_by_ttl: Mapping[str, str] = MappingProxyType( - {ttl: get_service_tier_cost_key(key, service_tier) for ttl, key in _TTL_PRICED_CACHE_WRITE_RATE_KEYS.items()} - ) - cache_read_cost = get_cost_per_unit(model_info, cache_read_cost_key, default_value=None) - - ## CHECK IF ABOVE THRESHOLD - # Optimization: collect threshold keys first to avoid sorting all model_info keys. - # Standard thresholds and thresholds suffixed for this request's service tier both count. - tier_key_suffix: Final = get_service_tier_cost_key("", service_tier) - threshold_prefixes: Final = tuple(f"{prefix}_above_" for prefix in _BATCH_RATE_PREFIXES) - threshold_keys: Final = [ - k - for k in model_info - if k.startswith(threshold_prefixes) - and "_tokens" in k - and ( - not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES) or (tier_key_suffix != "" and k.endswith(tier_key_suffix)) - ) - ] - - # Only sort the threshold keys (typically 1-2 keys instead of 66+) - threshold: float | None = None - for key in sorted(threshold_keys, key=_parse_above_token_threshold): - value = model_info.get(key) - if value is not None: - try: - # Handle both formats: _above_128k_tokens and _above_128_tokens - threshold_str = key.rsplit("_above_", 1)[1].split("_tokens")[0] - threshold = _parse_above_token_threshold(key) - if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive): - # Prefer a service_tier-specific above-threshold key when available, - # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini - # ON_DEMAND_PRIORITY. Falls back to the standard key automatically - # via _get_cost_per_unit's service_tier fallback logic. - tiered_input_key = ( - get_service_tier_cost_key( - f"input_cost_per_token_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"input_cost_per_token_above_{threshold_str}_tokens" - ) - prompt_base_cost = cast( - float, - get_cost_per_unit(model_info, tiered_input_key, prompt_base_cost), - ) - tiered_output_key = ( - get_service_tier_cost_key( - f"output_cost_per_token_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"output_cost_per_token_above_{threshold_str}_tokens" - ) - completion_base_cost = cast( - float, - get_cost_per_unit( - model_info, - tiered_output_key, - completion_base_cost, - ), - ) - - # Apply tiered pricing to cache costs - cache_creation_tiered_key = ( - get_service_tier_cost_key( - f"cache_creation_input_token_cost_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_creation_input_token_cost_above_{threshold_str}_tokens" - ) - cache_read_tiered_key = ( - get_service_tier_cost_key( - f"cache_read_input_token_cost_above_{threshold_str}_tokens", - service_tier, - ) - if service_tier - else f"cache_read_input_token_cost_above_{threshold_str}_tokens" - ) - - cache_creation_cost = get_cost_per_unit(model_info, cache_creation_tiered_key, cache_creation_cost) - - cache_creation_keys_by_ttl = MappingProxyType( - { - ttl: _first_present_key( - model_info, - get_service_tier_cost_key( - f"{_TTL_PRICED_CACHE_WRITE_RATE_KEYS[ttl]}_above_{threshold_str}_tokens", - service_tier, - ), - key, - ) - for ttl, key in cache_creation_keys_by_ttl.items() - } - ) - cache_read_cost = get_cost_per_unit(model_info, cache_read_tiered_key, cache_read_cost) - except (IndexError, ValueError): - continue - except Exception: - continue - - input_rate_for_missing_cache_rates: Final = _off_peak_rate( - _open_off_peak_block(model_info, current_time) or MappingProxyType({}), - "input_cost_per_token", - prompt_base_cost, - ) - resolved_cache_read_cost: Final = input_rate_for_missing_cache_rates if cache_read_cost is None else cache_read_cost - resolved_cache_creation_cost: Final = ( - input_rate_for_missing_cache_rates if cache_creation_cost is None else cache_creation_cost - ) - base_rates: Final = apply_off_peak_pricing( - model_info, - current_time, - TokenRates( - input_rate=prompt_base_cost, - output_rate=completion_base_cost, - cache_read_rate=resolved_cache_read_cost, - cache_creation_rate=resolved_cache_creation_cost, - reasoning_rate=None, - ), - ) - return replace( - base_rates, - cache_creation_rates_by_ttl=MappingProxyType( - { - ttl: base_rates.cache_creation_rate - if (rate := get_cost_per_unit(model_info, key, default_value=None)) is None - else rate - for ttl, key in cache_creation_keys_by_ttl.items() - } - ), - ) - - def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value: float | None) -> float: """ Generic cost calculator for any usage component @@ -1001,33 +391,6 @@ def resolve_image_model_info(model: str, custom_llm_provider: str, model_info: M return resolved -def calculate_cache_writing_cost( - cache_creation_tokens: int, - cache_creation_token_details: CacheCreationTokenDetails | None, - cache_creation_cost: float, - cache_creation_cost_by_ttl: Mapping[str, float], -) -> float: - """ - Bill cache writes per lifetime: Anthropic reports 5m and 1h counts, OpenAI reports 30m and 12h counts. - Writes no lifetime count covers, such as a lifetime a provider adds later, bill at the base write rate. - """ - if cache_creation_token_details is None: - return cache_creation_tokens * cache_creation_cost - tokens_by_ttl: Final[Mapping[str, int | None]] = MappingProxyType( - { - "5m": cache_creation_token_details.ephemeral_5m_input_tokens, - "30m": cache_creation_token_details.ephemeral_30m_input_tokens, - "1hr": cache_creation_token_details.ephemeral_1h_input_tokens, - "12hr": cache_creation_token_details.ephemeral_12h_input_tokens, - } - ) - unattributed_tokens: Final = max(cache_creation_tokens - sum(t or 0 for t in tokens_by_ttl.values()), 0) - return unattributed_tokens * cache_creation_cost + sum( - (tokens or 0) * cache_creation_cost_by_ttl.get(ttl, cache_creation_cost) - for ttl, tokens in tokens_by_ttl.items() - ) - - class PromptTokensDetailsResult(TypedDict): cache_hit_tokens: int cache_hit_audio_tokens: ReadOnly[int] @@ -1185,143 +548,6 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu ) -def _modality_token_cost( - model_info: ModelInfo, - modality_cost_key: str, - tokens: int, - prompt_base_cost: float, - service_tier: str | None, -) -> float: - """ - Audio, image and video prompt tokens bill at their modality rate when the model has one. - Without one they are ordinary prompt tokens, so they bill at `prompt_base_cost`, which - `_get_token_base_cost` has already resolved for the service tier and long-context tier. - """ - tiered_key: Final = get_service_tier_cost_key(modality_cost_key, service_tier) - cost_key: Final = tiered_key if model_info.get(tiered_key) is not None else modality_cost_key - if model_info.get(cost_key) is None: - return float(tokens) * prompt_base_cost - return calculate_cost_component(model_info, cost_key, tokens) - - -def _calculate_input_cost( - prompt_tokens_details: PromptTokensDetailsResult, - model_info: ModelInfo, - prompt_base_cost: float, - cache_read_cost: float, - cache_creation_cost: float, - cache_creation_cost_by_ttl: Mapping[str, float], - service_tier: str | None = None, -) -> float: - """ - Calculates the input cost for a given model, prompt tokens, and completion tokens. - - A model priced per character (`input_cost_per_character`) bills the reported character count - in place of its text tokens, so the same prompt is never charged under both units. - """ - bills_characters: Final = ( - prompt_tokens_details["character_count"] > 0 and model_info.get("input_cost_per_character") is not None - ) - prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost - - ### CACHE READ COST - Now uses tiered pricing - cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] - audio_cache_read_rate: Final = get_cost_per_unit( - model_info, - get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), - None, - ) - prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost - prompt_cost += float(cache_hit_audio_tokens) * ( - audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost - ) - - ### AUDIO COST - if prompt_tokens_details["audio_tokens"] and not ( - prompt_tokens_details["audio_length_seconds"] and model_info.get("input_cost_per_audio_per_second") is not None - ): - prompt_cost += _modality_token_cost( - model_info, - "input_cost_per_audio_token", - prompt_tokens_details["audio_tokens"], - prompt_base_cost, - service_tier, - ) - - ### IMAGE TOKEN COST - if prompt_tokens_details["image_tokens"] and not ( - prompt_tokens_details["image_count"] and model_info.get("input_cost_per_image") is not None - ): - prompt_cost += _modality_token_cost( - model_info, - "input_cost_per_image_token", - prompt_tokens_details["image_tokens"], - prompt_base_cost, - service_tier, - ) - - ### VIDEO TOKEN COST - if prompt_tokens_details["video_tokens"] and not ( - prompt_tokens_details["video_length_seconds"] and model_info.get("input_cost_per_video_per_second") is not None - ): - prompt_cost += _modality_token_cost( - model_info, - "input_cost_per_video_token", - prompt_tokens_details["video_tokens"], - prompt_base_cost, - service_tier, - ) - - ### CACHE WRITING COST - Now uses tiered pricing - if ( - prompt_tokens_details["cache_creation_tokens"] - or prompt_tokens_details["cache_creation_token_details"] is not None - ): - prompt_cost += calculate_cache_writing_cost( - cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"], - cache_creation_token_details=prompt_tokens_details["cache_creation_token_details"], - cache_creation_cost=cache_creation_cost, - cache_creation_cost_by_ttl=cache_creation_cost_by_ttl, - ) - - ### CHARACTER COST - if prompt_tokens_details["character_count"]: - prompt_cost += calculate_cost_component( - model_info, - "input_cost_per_character", - prompt_tokens_details["character_count"], - ) - - ### IMAGE COUNT COST - if prompt_tokens_details["image_count"]: - prompt_cost += calculate_cost_component( - model_info, "input_cost_per_image", prompt_tokens_details["image_count"] - ) - - ### VIDEO LENGTH COST - if prompt_tokens_details["video_length_seconds"]: - prompt_cost += calculate_cost_component( - model_info, - "input_cost_per_video_per_second", - prompt_tokens_details["video_length_seconds"], - ) - - ### AUDIO LENGTH COST - if prompt_tokens_details["audio_length_seconds"]: - prompt_cost += calculate_cost_component( - model_info, - "input_cost_per_audio_per_second", - prompt_tokens_details["audio_length_seconds"], - ) - - if prompt_tokens_details["query_count"]: - prompt_cost += calculate_cost_component( - model_info, "input_cost_per_query", prompt_tokens_details["query_count"] - ) - - return prompt_cost - - def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float: """ Resolve the per-model regional-processing uplift multiplier for a given @@ -1407,46 +633,6 @@ def get_provider_specific_speed_multiplier(model_info: ModelInfo, usage: Usage) return float(provider_specific_entry.get("fast", 1.0)) -def _resolve_reasoning_token_cost( - model_info: ModelInfo, - service_tier: str | None, - completion_base_cost: float, -) -> float: - tier_reasoning_key: Final = get_service_tier_cost_key("output_cost_per_reasoning_token", service_tier) - if model_info.get(tier_reasoning_key) is not None: - tier_reasoning_cost: Final = get_cost_per_unit(model_info, tier_reasoning_key, None) - if tier_reasoning_cost is not None: - return tier_reasoning_cost - tier_output_key: Final = get_service_tier_cost_key("output_cost_per_token", service_tier) - if tier_output_key != "output_cost_per_token" and model_info.get(tier_output_key) is not None: - return completion_base_cost - standard_reasoning_cost: Final = get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None) - return standard_reasoning_cost if standard_reasoning_cost is not None else completion_base_cost - - -def _resolve_billed_reasoning_rate( - model_info: ModelInfo, - usage: Usage, - service_tier: str | None, - completion_base_cost: float, - current_time: datetime | None, -) -> float: - off_peak: Final = _open_off_peak_block(model_info, current_time) - off_peak_reasoning_rate: Final = ( - None if off_peak is None else _parse_off_peak_rate(off_peak.get("output_cost_per_reasoning_token")) - ) - if off_peak_reasoning_rate is not None: - return off_peak_reasoning_rate - tiered_reasoning_rate: Final = _get_tiered_reasoning_rate(model_info=model_info, usage=usage) - if tiered_reasoning_rate is not None: - return tiered_reasoning_rate - return _resolve_reasoning_token_cost( - model_info=model_info, - service_tier=service_tier, - completion_base_cost=completion_base_cost, - ) - - def generic_cost_per_token( model: str, usage: Usage, @@ -1481,179 +667,77 @@ def generic_cost_per_token( # rather than handing back a name for this to re-resolve. A name cannot express a # per-deployment override: those are registered under the deployment id and kept off # the shared model-name key, so resolving from the name here reads the public rate. - resolved_model_info: Final = apply_provider_cache_read_default( - get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_info is None else model_info, - custom_llm_provider, + resolved_model_info: Final = ( + get_model_info(model=model, custom_llm_provider=custom_llm_provider) if model_info is None else model_info ) - ## CALCULATE INPUT COST - ### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing) - prompt_cost = 0.0 - ### PROCESSING COST - prompt_tokens_details = PromptTokensDetailsResult( - cache_hit_tokens=0, - cache_hit_audio_tokens=0, - cache_creation_tokens=0, - cache_creation_token_details=None, - text_tokens=usage.prompt_tokens, - audio_tokens=0, - image_tokens=0, - video_tokens=0, - character_count=0, - image_count=0, - video_length_seconds=0.0, - audio_length_seconds=0.0, - query_count=0, + factors: Final = token_pricing_factors( + resolved_model_info, usage, custom_llm_provider, data_residency, vertex_location + ) + return price_token_usage( + model, usage, custom_llm_provider, resolved_model_info, service_tier, current_time, factors ) - if usage.prompt_tokens_details: - prompt_tokens_details = parse_prompt_tokens_details(usage) - ## EDGE CASE - text tokens not set or includes cached tokens (double-counting) - ## Some providers (like xAI) report text_tokens = prompt_tokens (including cached) - ## We detect this when: text_tokens + cached_tokens + other > prompt_tokens - ## Ref: https://github.com/BerriAI/litellm/issues/19680, #14874, #14875 - cache_hit: Final = prompt_tokens_details["cache_hit_tokens"] - text_tokens = prompt_tokens_details["text_tokens"] - audio_tokens = prompt_tokens_details["audio_tokens"] - cache_creation: Final = prompt_tokens_details["cache_creation_tokens"] - image_tokens = prompt_tokens_details["image_tokens"] - video_tokens = prompt_tokens_details["video_tokens"] - - # Check for double-counting: sum of details > prompt_tokens means overlap - total_details: Final = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens - has_double_counting: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens - - if has_double_counting: - # cached and per-modality counts are both subsets of prompt_tokens and may overlap, so a - # modality can only bill what the cache did not already cover or the overlap is billed twice - uncached_budget: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0) - billable_audio: Final = min(audio_tokens, uncached_budget) - billable_image: Final = min(image_tokens, uncached_budget - billable_audio) - billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image) - prompt_tokens_details["audio_tokens"] = billable_audio - prompt_tokens_details["image_tokens"] = billable_image - prompt_tokens_details["video_tokens"] = billable_video - prompt_tokens_details["text_tokens"] = uncached_budget - billable_audio - billable_image - billable_video - elif text_tokens == 0 and prompt_tokens_details["image_count"] == 0: - # Clamp to zero: inconsistent streaming usage - prompt_tokens_details["text_tokens"] = max( - usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0 - ) +def price_token_usage( + model: str, + usage: Usage, + custom_llm_provider: str, + resolved_model_info: ModelInfo, + service_tier: str | None, + current_time: datetime | None, + factors: Sequence[float], +) -> tuple[float, float]: + from litellm._cost import billing_context, price, require_priced, token_usage billing_time: Final = current_time if current_time is not None else current_billing_time() - base_costs: Final = _get_token_base_cost( - model_info=resolved_model_info, - usage=usage, - service_tier=service_tier, - current_time=billing_time, - threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), + priced: Final = require_priced( + price( + {"litellm_provider": custom_llm_provider, **resolved_model_info}, + token_usage( + usage, + resolved_model_info, + inclusive_threshold=uses_inclusive_token_thresholds(custom_llm_provider), + ), + billing_context(billing_time, service_tier), + factors=factors, + cache_read_default_ratio=cache_read_default_ratio(custom_llm_provider), + ), + model, ) - completion_base_cost: Final = base_costs.output_rate - resolved_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate( - model_info=resolved_model_info, - usage=usage, - current_time=billing_time, - service_tier=service_tier, + details: Final = parse_prompt_tokens_details(usage) + non_token_input: Final = sum( + calculate_cost_component(resolved_model_info, key, quantity) + for key, quantity in ( + ("input_cost_per_character", details["character_count"]), + ("input_cost_per_image", details["image_count"]), + ("input_cost_per_video_per_second", details["video_length_seconds"]), + ("input_cost_per_audio_per_second", details["audio_length_seconds"]), + ("input_cost_per_query", details["query_count"]), + ) ) - effective_cache_read_cost: Final = ( - base_costs.cache_read_rate if resolved_cache_read_cost is None else resolved_cache_read_cost + return priced.input + non_token_input * prod(factors), priced.output + + +def token_pricing_factors( + model_info: ModelInfo, + usage: Usage, + custom_llm_provider: str | None, + data_residency: str | None, + vertex_location: str | None, +) -> tuple[float, ...]: + regional: Final = ( + get_regional_uplift_multiplier(model_info, data_residency), + get_vertex_regional_endpoint_uplift(model_info, vertex_location), ) - - prompt_cost = _calculate_input_cost( - prompt_tokens_details=prompt_tokens_details, - model_info=resolved_model_info, - prompt_base_cost=base_costs.input_rate, - cache_read_cost=effective_cache_read_cost, - cache_creation_cost=base_costs.cache_creation_rate, - cache_creation_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, - service_tier=service_tier, + if custom_llm_provider != "anthropic": + return regional + return ( + *regional, + get_provider_specific_speed_multiplier(model_info, usage), + get_provider_specific_geo_multiplier(model_info, usage), ) - ## CALCULATE OUTPUT COST - text_tokens = 0 - audio_tokens = 0 - reasoning_tokens = 0 - image_tokens = 0 - video_tokens = 0 - is_text_tokens_total = False - if usage.completion_tokens_details is not None: - completion_tokens_details: Final = parse_completion_tokens_details(usage) - audio_tokens = completion_tokens_details["audio_tokens"] - text_tokens = completion_tokens_details["text_tokens"] - reasoning_tokens = completion_tokens_details["reasoning_tokens"] - image_tokens = completion_tokens_details["image_tokens"] - video_tokens = completion_tokens_details["video_tokens"] - - # Handle text_tokens calculation: - # 1. If text_tokens is explicitly provided and > 0, use it - # 2. If there's a breakdown (reasoning/audio/image/video tokens), calculate text_tokens as the remainder - # 3. If no breakdown at all, assume all completion_tokens are text_tokens - has_token_breakdown: Final = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0 or video_tokens > 0 - if text_tokens == 0: - if has_token_breakdown: - # Calculate text tokens as remainder when we have a breakdown - # This handles cases like OpenAI's reasoning models where text_tokens isn't provided - text_tokens = max( - 0, - usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens - video_tokens, - ) - else: - # No breakdown at all, all tokens are text tokens - text_tokens = usage.completion_tokens - is_text_tokens_total = True - ## TEXT COST - completion_cost = float(text_tokens) * completion_base_cost - - ## AUDIO COST - if not is_text_tokens_total and audio_tokens is not None and audio_tokens > 0: - _output_cost_per_audio_token = get_cost_per_unit(resolved_model_info, "output_cost_per_audio_token", None) - _output_cost_per_audio_token = ( - _output_cost_per_audio_token if _output_cost_per_audio_token is not None else completion_base_cost - ) - completion_cost += float(audio_tokens) * _output_cost_per_audio_token - - ## REASONING COST - if not is_text_tokens_total and reasoning_tokens and reasoning_tokens > 0: - completion_cost += float(reasoning_tokens) * _resolve_billed_reasoning_rate( - model_info=resolved_model_info, - usage=usage, - service_tier=service_tier, - completion_base_cost=completion_base_cost, - current_time=billing_time, - ) - - ## IMAGE COST - if not is_text_tokens_total and image_tokens and image_tokens > 0: - _output_cost_per_image_token = get_cost_per_unit(resolved_model_info, "output_cost_per_image_token", None) - _output_cost_per_image_token = ( - _output_cost_per_image_token if _output_cost_per_image_token is not None else completion_base_cost - ) - completion_cost += float(image_tokens) * _output_cost_per_image_token - - ## VIDEO COST - if not is_text_tokens_total and video_tokens and video_tokens > 0: - _output_cost_per_video_token = get_cost_per_unit(resolved_model_info, "output_cost_per_video_token", None) - _output_cost_per_video_token = ( - _output_cost_per_video_token if _output_cost_per_video_token is not None else completion_base_cost - ) - completion_cost += float(video_tokens) * _output_cost_per_video_token - - ## REGIONAL DATA-RESIDENCY UPLIFT - # Applied as a flat multiplier across all token costs for the request - # when the upstream is a regionalized OpenAI host (eu./us.api.openai.com). - uplift: Final = get_regional_uplift_multiplier(resolved_model_info, data_residency) - if uplift != 1.0: - prompt_cost *= uplift - completion_cost *= uplift - - vertex_uplift: Final = get_vertex_regional_endpoint_uplift(resolved_model_info, vertex_location) - if vertex_uplift != 1.0: - prompt_cost *= vertex_uplift - completion_cost *= vertex_uplift - - return prompt_cost, completion_cost - def _coerce_token_count(value: object) -> int: return value if isinstance(value, int) and value > 0 else 0 @@ -1661,31 +745,13 @@ def _coerce_token_count(value: object) -> int: @dataclass(frozen=True, slots=True) class BilledTokenRates: - """Per-token rates one request's usage bills at, after token tiers, off-peak windows and the - regional multipliers the totals apply, so each cost line equals its token count times its rate.""" - - input_cost_per_token: float - output_cost_per_token: float - cache_read_input_token_cost: float - cache_read_input_audio_token_cost: float - cache_creation_input_token_cost: float - cache_creation_input_token_cost_by_ttl: Mapping[str, float] - output_cost_per_reasoning_token: float - - def scaled(self, multiplier: float) -> "BilledTokenRates": - if multiplier == 1.0: - return self - return BilledTokenRates( - input_cost_per_token=self.input_cost_per_token * multiplier, - output_cost_per_token=self.output_cost_per_token * multiplier, - cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier, - cache_read_input_audio_token_cost=self.cache_read_input_audio_token_cost * multiplier, - cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier, - cache_creation_input_token_cost_by_ttl=MappingProxyType( - {ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()} - ), - output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, - ) + input_cost_per_token: float | None + output_cost_per_token: float | None + cache_read_input_token_cost: float | None + cache_read_input_audio_token_cost: float | None + cache_creation_input_token_cost: float | None + cache_creation_input_token_cost_by_ttl: Mapping[str, float | None] + output_cost_per_reasoning_token: float | None @dataclass(frozen=True, slots=True) @@ -1698,102 +764,83 @@ class TokenTypeCostBreakdown: differently-argued way. None when the model's pricing could not be resolved.""" -def _reasoning_token_count(usage: Usage) -> int: - parsed: Final = ( - parse_completion_tokens_details(usage)["reasoning_tokens"] if usage.completion_tokens_details is not None else 0 - ) - return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0)) +_BILLED_TOKEN_METERS: Final = ( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "explicit_cache_read_input_token_cost", + "cache_read_input_audio_token_cost", + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr", + "cache_creation_input_token_cost_above_12hr", + "output_cost_per_reasoning_token", +) -def _cache_token_counts(usage: Usage) -> tuple[int, int, int, CacheCreationTokenDetails | None]: - """(cache read tokens, cached audio tokens, cache creation tokens, cache creation details): read from - prompt_tokens_details first, then the private top-level counters the Usage constructor mirrors cache - tokens onto for providers/callers that bypass the details.""" - parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None - parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0 - parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0 - return ( - parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)), - parsed["cache_hit_audio_tokens"] if parsed is not None else 0, - parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)), - parsed["cache_creation_token_details"] if parsed is not None else None, - ) - - -def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRates: - """Flat custom pricing has no tiers, uplifts or reasoning rate: cache tokens bill at the configured - cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does.""" - input_rate: Final = custom_cost_per_token["input_cost_per_token"] - output_rate: Final = custom_cost_per_token["output_cost_per_token"] - cache_read_rate: Final = custom_cost_per_token.get("cache_read_input_token_cost", input_rate) - cache_creation_rate: Final = custom_cost_per_token.get("cache_creation_input_token_cost", input_rate) - return BilledTokenRates( - input_cost_per_token=input_rate, - output_cost_per_token=output_rate, - cache_read_input_token_cost=cache_read_rate, - cache_read_input_audio_token_cost=cache_read_rate, - cache_creation_input_token_cost=cache_creation_rate, - cache_creation_input_token_cost_by_ttl=MappingProxyType({}), - output_cost_per_reasoning_token=output_rate, - ) - - -def _cost_map_billed_rates( - model_info: ModelInfo, - usage: Usage, +def _reported_token_evaluation( + model: str, custom_llm_provider: str | None, + usage: Usage, service_tier: str | None, data_residency: str | None, vertex_location: str | None, current_time: datetime | None, -) -> BilledTokenRates: - billing_time: Final = current_time if current_time is not None else current_billing_time() - base_costs: Final = _get_token_base_cost( - model_info=model_info, - usage=usage, - service_tier=service_tier, - current_time=billing_time, - threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), + custom_cost_per_token: CostPerToken | None, + model_info: ModelInfo | None, +) -> _native.PriceEvaluation | _native.RowErrors | None: + from litellm._cost import billing_context, evaluate, token_usage + + if custom_cost_per_token is not None: + custom_row: Final = {"litellm_provider": "custom", **custom_cost_per_token} + return evaluate( + custom_row, + token_usage(usage, custom_row), + billing_context(current_time if current_time is not None else current_billing_time()), + project=_BILLED_TOKEN_METERS, + ) + try: + resolved_model_info: Final = ( + model_info + if model_info is not None + else get_model_info(model=model, custom_llm_provider=custom_llm_provider) + ) + except Exception: # noqa: BLE001 # preserve the public unknown-model reporting contract + return None + pricing_row: Final[Mapping[str, object]] = ( + resolved_model_info + if "litellm_provider" in resolved_model_info + else dict(resolved_model_info, litellm_provider=custom_llm_provider or "custom") ) - provider_cache_read_cost_rate: Final = resolve_explicit_cache_read_cost_rate( - model_info=model_info, - usage=usage, - current_time=billing_time, - service_tier=service_tier, - ) - effective_cache_read_cost_rate: Final = ( - base_costs.cache_read_rate if provider_cache_read_cost_rate is None else provider_cache_read_cost_rate + return evaluate( + pricing_row, + token_usage(usage, pricing_row, inclusive_threshold=uses_inclusive_token_thresholds(custom_llm_provider)), + billing_context(current_time if current_time is not None else current_billing_time(), service_tier), + project=_BILLED_TOKEN_METERS, + factors=token_pricing_factors(resolved_model_info, usage, custom_llm_provider, data_residency, vertex_location), + cache_read_default_ratio=cache_read_default_ratio(custom_llm_provider), ) - reasoning_rate: Final = _resolve_billed_reasoning_rate( - model_info=model_info, - usage=usage, - service_tier=service_tier, - completion_base_cost=base_costs.output_rate, - current_time=billing_time, - ) - audio_cache_read_rate: Final = get_cost_per_unit( - model_info, - get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), - None, - ) - multiplier: Final = ( - get_regional_uplift_multiplier(model_info, data_residency) - * get_vertex_regional_endpoint_uplift(model_info, vertex_location) - * get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) - * get_provider_specific_speed_multiplier(model_info=model_info, usage=usage) - ) + +def _billed_rates(evaluation: _native.PriceEvaluation, usage: Usage, *, flat_custom: bool = False) -> BilledTokenRates: + from litellm._cost import cache_read_meter + + rates: Final = {rate.requested_meter: rate.unit_price for rate in evaluation.rates} return BilledTokenRates( - input_cost_per_token=base_costs.input_rate, - output_cost_per_token=base_costs.output_rate, - cache_read_input_token_cost=effective_cache_read_cost_rate, - cache_read_input_audio_token_cost=( - audio_cache_read_rate if audio_cache_read_rate is not None else effective_cache_read_cost_rate + input_cost_per_token=rates["input_cost_per_token"], + output_cost_per_token=rates["output_cost_per_token"], + cache_read_input_token_cost=rates[cache_read_meter(usage)], + cache_read_input_audio_token_cost=rates["cache_read_input_audio_token_cost"], + cache_creation_input_token_cost=rates["cache_creation_input_token_cost"], + cache_creation_input_token_cost_by_ttl=MappingProxyType( + {} + if flat_custom + else { + "1hr": rates["cache_creation_input_token_cost_above_1hr"], + "12hr": rates["cache_creation_input_token_cost_above_12hr"], + } ), - cache_creation_input_token_cost=base_costs.cache_creation_rate, - cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, - output_cost_per_reasoning_token=reasoning_rate, - ).scaled(multiplier) + output_cost_per_reasoning_token=rates["output_cost_per_reasoning_token"], + ) def get_billed_token_rates( @@ -1807,28 +854,20 @@ def get_billed_token_rates( custom_cost_per_token: CostPerToken | None = None, model_info: ModelInfo | None = None, ) -> BilledTokenRates | None: - """Rates the cost calculator bills ``usage`` at, resolved exactly as the totals and the token-type - breakdown resolve them. None when the model's pricing cannot be resolved.""" - if custom_cost_per_token is not None: - return _custom_pricing_rates(custom_cost_per_token) - try: - resolved_model_info: Final = apply_provider_cache_read_default( - model_info - if model_info is not None - else get_model_info(model=model, custom_llm_provider=custom_llm_provider), - custom_llm_provider, - ) - except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model: no rates - return None - return _cost_map_billed_rates( - model_info=resolved_model_info, - usage=usage, - custom_llm_provider=custom_llm_provider, - service_tier=service_tier, - data_residency=data_residency, - vertex_location=vertex_location, - current_time=current_time, + evaluation: Final = _reported_token_evaluation( + model, + custom_llm_provider, + usage, + service_tier, + data_residency, + vertex_location, + current_time, + custom_cost_per_token, + model_info, ) + if not isinstance(evaluation, _native.PriceEvaluation): + return None + return _billed_rates(evaluation, usage, flat_custom=custom_cost_per_token is not None) def get_token_type_cost_breakdown( @@ -1842,55 +881,45 @@ def get_token_type_cost_breakdown( custom_cost_per_token: CostPerToken | None = None, model_info: ModelInfo | None = None, ) -> TokenTypeCostBreakdown: - """ - Provider-agnostic cost of reasoning and cache tokens, derived from the usage - object and model pricing alone. - - This works for every provider, including Perplexity/Cerebras/Dashscope whose - cost calculators bypass ``generic_cost_per_token``, because cache tokens always - land on ``prompt_tokens_details`` (via the Usage constructor and provider - transformations) and reasoning tokens on ``completion_tokens_details``. It reuses - the same rate resolution as the total-cost path (``get_billed_token_rates``) so the - breakdown can never drift from the totals. A deployment billed by - ``custom_cost_per_token`` is priced from those flat rates instead of the cost map and, - like its totals, bills cache writes flat rather than by their 5m/1h split. - Returns zeros (never raises) when the model or its pricing cannot be resolved. - """ - rates: Final = get_billed_token_rates( - model=model, - custom_llm_provider=custom_llm_provider, - usage=usage, - service_tier=service_tier, - data_residency=data_residency, - vertex_location=vertex_location, - current_time=current_time, - custom_cost_per_token=custom_cost_per_token, - model_info=model_info, + evaluation: Final = _reported_token_evaluation( + model, + custom_llm_provider, + usage, + service_tier, + data_residency, + vertex_location, + current_time, + custom_cost_per_token, + model_info, ) - if rates is None: + if not isinstance(evaluation, _native.PriceEvaluation) or not isinstance(evaluation.pricing, _native.Priced): return TokenTypeCostBreakdown(0.0, 0.0, 0.0) - - cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts( - usage - ) - cache_creation_cost: Final = ( - float(cache_creation_tokens) * rates.cache_creation_input_token_cost - if custom_cost_per_token is not None - else calculate_cache_writing_cost( - cache_creation_tokens=cache_creation_tokens, - cache_creation_token_details=cache_creation_token_details, - cache_creation_cost=rates.cache_creation_input_token_cost, - cache_creation_cost_by_ttl=rates.cache_creation_input_token_cost_by_ttl, - ) - ) + lines: Final = evaluation.pricing.lines return TokenTypeCostBreakdown( - reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token, - cache_read_cost=( - float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost - + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost + reasoning_cost=sum( + line.amount for line in lines if (line.fallback_from or line.meter) == "output_cost_per_reasoning_token" ), - cache_creation_cost=cache_creation_cost, - rates=rates, + cache_read_cost=sum( + line.amount + for line in lines + if (line.fallback_from or line.meter) + in ( + "cache_read_input_token_cost", + "explicit_cache_read_input_token_cost", + "cache_read_input_audio_token_cost", + ) + ), + cache_creation_cost=sum( + line.amount + for line in lines + if (line.fallback_from or line.meter) + in ( + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr", + "cache_creation_input_token_cost_above_12hr", + ) + ), + rates=_billed_rates(evaluation, usage, flat_custom=custom_cost_per_token is not None), ) @@ -1910,52 +939,60 @@ def calculate_prompt_caching_savings( ``billed_at`` is the request's completion time, so off-peak windows resolve as the biller saw them rather than at the later spend write. """ - model_info_with_cache_read_default: Final = apply_provider_cache_read_default(model_info, custom_llm_provider) - base_costs: Final = _get_token_base_cost( - model_info=model_info_with_cache_read_default, - usage=usage, - service_tier=service_tier, - current_time=billed_at, - threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider), + from litellm._cost import billing_context, cache_read_meter, cache_write_quantities, price + + row: Final = {"litellm_provider": custom_llm_provider or "custom", **model_info} + when: Final = billing_context(billed_at if billed_at is not None else current_billing_time(), service_tier) + factors: Final = ( + get_regional_uplift_multiplier(model_info, data_residency), + get_vertex_regional_endpoint_uplift(model_info, vertex_location), ) - provider_cache_read_cost: Final = resolve_explicit_cache_read_cost_rate( - model_info=model_info, - usage=usage, - current_time=billed_at, - service_tier=service_tier, - ) - prompt_base_cost: Final = base_costs.input_rate - effective_cache_read_cost: Final = ( - base_costs.cache_read_rate if provider_cache_read_cost is None else provider_cache_read_cost - ) - write_rate: Final = base_costs.cache_creation_rate or prompt_base_cost - prompt_tokens_details: Final = parse_prompt_tokens_details(usage) - cache_read_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"], 0) - cache_creation_tokens: Final = max(prompt_tokens_details["cache_creation_tokens"], 0) - details: Final = prompt_tokens_details["cache_creation_token_details"] - cache_creation_details: Final = ( - CacheCreationTokenDetails( - ephemeral_5m_input_tokens=max(details.ephemeral_5m_input_tokens or 0, 0), - ephemeral_30m_input_tokens=max(details.ephemeral_30m_input_tokens or 0, 0), - ephemeral_1h_input_tokens=max(details.ephemeral_1h_input_tokens or 0, 0), - ephemeral_12h_input_tokens=max(details.ephemeral_12h_input_tokens or 0, 0), + + def cache_cost(quantities: Mapping[str, int]) -> _native.Priced | None: + result: Final = price( + row, + _native.UsageInput( + dict(quantities), + prompt_tokens=usage.prompt_tokens, + inclusive_threshold=uses_inclusive_token_thresholds(custom_llm_provider), + ), + when, + factors=factors, + cache_read_default_ratio=cache_read_default_ratio(custom_llm_provider), ) - if details is not None - else None + return result if isinstance(result, _native.Priced) else None + + details: Final = parse_prompt_tokens_details(usage) + reads: Final = max(details["cache_hit_tokens"], 0) + ttl: Final = details["cache_creation_token_details"] + writes: Final = cache_write_quantities( + max(details["cache_creation_tokens"], 0), + CacheCreationTokenDetails( + ephemeral_5m_input_tokens=max(ttl.ephemeral_5m_input_tokens or 0, 0), + ephemeral_30m_input_tokens=max(ttl.ephemeral_30m_input_tokens or 0, 0), + ephemeral_1h_input_tokens=max(ttl.ephemeral_1h_input_tokens or 0, 0), + ephemeral_12h_input_tokens=max(ttl.ephemeral_12h_input_tokens or 0, 0), + ) + if ttl is not None + else None, ) - read_discount: Final = cache_read_tokens * max(prompt_base_cost - effective_cache_read_cost, 0.0) - write_premium: Final = calculate_cache_writing_cost( - cache_creation_tokens=cache_creation_tokens, - cache_creation_token_details=cache_creation_details, - cache_creation_cost=write_rate - prompt_base_cost, - cache_creation_cost_by_ttl=MappingProxyType( - {ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()} - ), + read_meter: Final = cache_read_meter(usage) + actual: Final = cache_cost({read_meter: reads, **writes}) + if actual is None: + return 0.0 + priced_writes: Final = tuple( + line + for line in actual.lines + if (line.fallback_from or line.meter) in writes + and ((line.fallback_from or line.meter) != "cache_creation_input_token_cost" or line.unit_price > 0) ) - uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift( - model_info, vertex_location - ) - return (read_discount - write_premium) * uplift + read_baseline: Final = cache_cost({"input_cost_per_token": reads}) + write_baseline: Final = cache_cost({"input_cost_per_token": sum(line.quantity for line in priced_writes)}) + if read_baseline is None or write_baseline is None: + return 0.0 + read_cost: Final = sum(line.amount for line in actual.lines if (line.fallback_from or line.meter) == read_meter) + write_cost: Final = sum(line.amount for line in priced_writes) + return max(read_baseline.input - read_cost, 0.0) - (write_cost - write_baseline.input) def calculate_image_response_cost_from_usage( diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 94c8149eff3..84bb607e63c 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -9,15 +9,12 @@ from pydantic import ValidationError from litellm.litellm_core_utils.llm_cost_calc.utils import ( generic_cost_per_token, - get_provider_specific_geo_multiplier, - get_provider_specific_speed_multiplier, get_web_search_requests_from_usage, ) from litellm.types.llms.base import LiteLLMBaseModel if TYPE_CHECKING: from litellm.types.utils import ModelInfo, Usage -import litellm def cost_per_token( @@ -36,7 +33,7 @@ def cost_per_token( Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ - prompt_cost, completion_cost = generic_cost_per_token( + return generic_cost_per_token( model=model, usage=usage, custom_llm_provider="anthropic", @@ -44,28 +41,6 @@ def cost_per_token( model_info=model_info, ) - # Apply provider_specific_entry multipliers for geo/speed routing - try: - effective_info: Final = ( - model_info - if model_info is not None - else litellm.get_model_info(model=model, custom_llm_provider="anthropic") - ) - geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=effective_info, usage=usage) - speed_multiplier: Final = get_provider_specific_speed_multiplier(model_info=effective_info, usage=usage) - - if speed_multiplier != 1.0: - prompt_cost *= speed_multiplier - completion_cost *= speed_multiplier - - if geo_multiplier != 1.0: - prompt_cost *= geo_multiplier - completion_cost *= geo_multiplier - except Exception: - pass - - return prompt_cost, completion_cost - class _AnthropicServerToolUseProbe(LiteLLMBaseModel): web_search_requests: int | None = None diff --git a/litellm/llms/azure_ai/cost_calculator.py b/litellm/llms/azure_ai/cost_calculator.py index 5934525eca3..da82feb2a90 100644 --- a/litellm/llms/azure_ai/cost_calculator.py +++ b/litellm/llms/azure_ai/cost_calculator.py @@ -5,8 +5,11 @@ Handles Azure AI Foundry Model Router flat cost and other Azure AI specific pric from typing import Final +from litellm._cost import billing_context, price, require_priced +from litellm._internal_context import current_billing_time from litellm._logging import verbose_logger from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.rust_bridge import _native from litellm.types.utils import Usage from litellm.utils import get_model_info @@ -57,10 +60,14 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl if not is_azure_model_router(model): return 0.0 model_info: Final = get_model_info(model=_router_fee_entry_name(model), custom_llm_provider="azure_ai") - router_flat_cost_per_token: Final = model_info.get("input_cost_per_token", 0) - if router_flat_cost_per_token and router_flat_cost_per_token > 0: - return prompt_tokens * router_flat_cost_per_token - return 0.0 + return require_priced( + price( + model_info, + _native.UsageInput({"input_cost_per_token": prompt_tokens}, prompt_tokens=prompt_tokens), + billing_context(current_billing_time()), + ), + model, + ).input def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float]: diff --git a/litellm/llms/bedrock/cost_calculation.py b/litellm/llms/bedrock/cost_calculation.py index 1b45531dda4..a1176ef2fba 100644 --- a/litellm/llms/bedrock/cost_calculation.py +++ b/litellm/llms/bedrock/cost_calculation.py @@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Final import litellm from litellm.litellm_core_utils.llm_cost_calc.utils import ( - generic_cost_per_token, get_provider_specific_geo_multiplier, + price_token_usage, ) if TYPE_CHECKING: @@ -22,12 +22,12 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) Follows the same logic as Anthropic's cost per token calculation. """ model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="bedrock") - prompt_cost, completion_cost = generic_cost_per_token( + return price_token_usage( model=model, usage=usage, custom_llm_provider="bedrock", service_tier=service_tier, - model_info=model_info, + resolved_model_info=model_info, + current_time=None, + factors=(get_provider_specific_geo_multiplier(model_info=model_info, usage=usage),), ) - geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage) - return prompt_cost * geo_multiplier, completion_cost * geo_multiplier diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py index 55d6fb1a0e2..3398f8d7159 100644 --- a/litellm/llms/dashscope/cost_calculator.py +++ b/litellm/llms/dashscope/cost_calculator.py @@ -1,113 +1,7 @@ -""" -Cost calculator for Dashscope Chat models. - -Alibaba Model Studio tiered pricing is all-or-nothing: the tier is picked from the -total input tokens of a single request, and every token of that request (input, -cached, cache-creation, output, reasoning) is billed at that one tier's rate. -See https://help.aliyun.com/zh/model-studio/billing-for-model-studio -""" - -from collections.abc import Mapping -from dataclasses import dataclass, replace from datetime import datetime -from typing import Final -from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate -from litellm.litellm_core_utils.llm_cost_calc.utils import ( - TokenRates, - apply_off_peak_pricing, - parse_completion_tokens_details, - parse_prompt_tokens_details, - resolve_explicit_cache_read_cost_rate, -) -from litellm.types.utils import ModelInfo, Usage -from litellm.utils import get_model_info - - -@dataclass(frozen=True, slots=True) -class TokenBreakdown: - text_tokens: int - cached_tokens: int - cache_creation_tokens: int - completion_tokens: int - reasoning_tokens: int - - @property - def total_input_tokens(self) -> int: - return self.text_tokens + self.cached_tokens + self.cache_creation_tokens - - -def _extract_token_breakdown(usage: Usage) -> TokenBreakdown: - prompt_details: Final = parse_prompt_tokens_details(usage) - cached_tokens: Final = prompt_details["cache_hit_tokens"] - cache_creation_tokens: Final = prompt_details["cache_creation_tokens"] - text_tokens: Final = max(usage.prompt_tokens - cached_tokens - cache_creation_tokens, 0) - - reasoning_tokens: Final = parse_completion_tokens_details(usage)["reasoning_tokens"] - completion_tokens: Final = max((usage.completion_tokens or 0) - reasoning_tokens, 0) - - return TokenBreakdown( - text_tokens=text_tokens, - cached_tokens=cached_tokens, - cache_creation_tokens=cache_creation_tokens, - completion_tokens=completion_tokens, - reasoning_tokens=reasoning_tokens, - ) - - -def _flat_rate(value: float | None, fallback_value: float | None) -> float: - if value is None: - return float(fallback_value or 0.0) - return float(value) - - -def _flat_rates(model_info: ModelInfo) -> TokenRates: - reasoning_rate: Final = model_info.get("output_cost_per_reasoning_token") - return TokenRates( - input_rate=float(model_info.get("input_cost_per_token") or 0.0), - cache_read_rate=_flat_rate( - model_info.get("cache_read_input_token_cost"), model_info.get("input_cost_per_token") - ), - cache_creation_rate=_flat_rate( - model_info.get("cache_creation_input_token_cost"), model_info.get("input_cost_per_token") - ), - output_rate=float(model_info.get("output_cost_per_token") or 0.0), - reasoning_rate=None if reasoning_rate is None else float(reasoning_rate), - ) - - -def _tier_rates(model_info: ModelInfo, tier: Mapping[str, object]) -> TokenRates: - # A tier that declares output rates keeps the request on them, all-or-nothing. A tier table - # spelling out only input rates would serve every completion for free, so there the model's - # own output rates stand in - flat_rates: Final = _flat_rates(model_info) - tier_declares_output: Final = "output_cost_per_token" in tier - tier_declares_reasoning: Final = "output_cost_per_reasoning_token" in tier - return TokenRates( - input_rate=tier_rate(tier, "input_cost_per_token"), - cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"), - cache_creation_rate=tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token"), - output_rate=tier_rate(tier, "output_cost_per_token") if tier_declares_output else flat_rates.output_rate, - reasoning_rate=( - tier_rate(tier, "output_cost_per_reasoning_token") - if tier_declares_reasoning - else None - if tier_declares_output - else flat_rates.reasoning_rate - ), - ) - - -def _bill(breakdown: TokenBreakdown, rates: TokenRates) -> tuple[float, float]: - prompt_cost: Final = ( - (breakdown.text_tokens * rates.input_rate) - + (breakdown.cached_tokens * rates.cache_read_rate) - + (breakdown.cache_creation_tokens * rates.cache_creation_rate) - ) - completion_cost: Final = (breakdown.completion_tokens * rates.output_rate) + ( - breakdown.reasoning_tokens * rates.billed_reasoning_rate - ) - return prompt_cost, completion_cost +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token +from litellm.types.utils import Usage def cost_per_token( @@ -116,39 +10,9 @@ def cost_per_token( custom_llm_provider: str = "dashscope", current_time: datetime | None = None, ) -> tuple[float, float]: - """ - Calculate cost per token for Dashscope models. - - Supports both tiered and flat pricing with cached and reasoning tokens, and swaps in the - model's off_peak_pricing rates while one of its windows is open. - - Args: - model: Model name without provider prefix - usage: LiteLLM Usage block - custom_llm_provider: The provider id the request resolved to; dashscope or one of its brand aliases - current_time: The moment the request is billed at; defaults to now, UTC - - Returns: - Tuple[float, float] - (prompt_cost_in_usd, completion_cost_in_usd) - """ - model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) - breakdown: Final = _extract_token_breakdown(usage) - raw_tiers: Final = model_info.get("tiered_pricing") - tiered_pricing: Final = raw_tiers if isinstance(raw_tiers, list) else None - tier: Final = ( - select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=breakdown.total_input_tokens) - if tiered_pricing - else None - ) - standard_rates: Final = _flat_rates(model_info) if tier is None else _tier_rates(model_info, tier) - rates: Final = apply_off_peak_pricing(model_info, current_time, standard_rates) - explicit_cache_read_rate: Final = resolve_explicit_cache_read_cost_rate( - model_info=model_info, + return generic_cost_per_token( + model=model, usage=usage, + custom_llm_provider=custom_llm_provider, current_time=current_time, ) - billed_rates: Final = ( - rates if explicit_cache_read_rate is None else replace(rates, cache_read_rate=explicit_cache_read_rate) - ) - - return _bill(breakdown, billed_rates) diff --git a/litellm/llms/fireworks_ai/cache_pricing.py b/litellm/llms/fireworks_ai/cache_pricing.py deleted file mode 100644 index f5e49cad01a..00000000000 --- a/litellm/llms/fireworks_ai/cache_pricing.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import ( - Final, - cast, # noqa: TID251 # the derived entry is a dict copy of a ReadOnly TypedDict; no cast-free way to retype it -) - -from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO -from litellm.types.utils import ModelInfo - - -def _as_rate(value: object) -> float | None: - if isinstance(value, bool) or not isinstance(value, (int, float, str)): - return None - try: - return float(value) - except ValueError: - return None - - -def with_default_cache_read_rate(model_info: ModelInfo) -> ModelInfo: - input_rate: Final = _as_rate(model_info.get("input_cost_per_token")) - if model_info.get("cache_read_input_token_cost") is not None or input_rate is None: - return model_info - cache_read_rate: Final = input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO - off_peak: Final = model_info.get("off_peak_pricing") - if off_peak is None or "cache_read_input_token_cost" in off_peak: - return cast(ModelInfo, {**model_info, "cache_read_input_token_cost": cache_read_rate}) - off_peak_input_rate: Final = _as_rate(off_peak.get("input_cost_per_token")) - return cast( - ModelInfo, - { - **model_info, - "cache_read_input_token_cost": cache_read_rate, - "off_peak_pricing": { - **off_peak, - "cache_read_input_token_cost": ( - off_peak_input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO - if off_peak_input_rate is not None - else cache_read_rate - ), - }, - }, - ) diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 2d762adf958..81572794872 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -6,7 +6,9 @@ Helper util for handling perplexity-specific cost calculation from datetime import datetime from typing import Final -from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing +from litellm._cost import billing_context, price, require_priced +from litellm._internal_context import current_billing_time +from litellm.rust_bridge import _native from litellm.types.utils import Usage from litellm.utils import get_model_info @@ -52,22 +54,31 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non except (ValueError, TypeError): return default - rates: Final = apply_off_peak_pricing( - model_info, - current_time, - TokenRates( - input_rate=_safe_float_cast(model_info.get("input_cost_per_token")), - output_rate=_safe_float_cast(model_info.get("output_cost_per_token")), - cache_read_rate=0.0, - cache_creation_rate=0.0, - reasoning_rate=None, - ), + top_reasoning: Final[object] = getattr(usage, "reasoning_tokens", None) + reasoning_tokens: Final = ( + top_reasoning + if isinstance(top_reasoning, int) and top_reasoning > 0 + else (usage.completion_tokens_details.reasoning_tokens or 0) + if usage.completion_tokens_details is not None + else 0 ) - input_cost_per_token: Final = rates.input_rate - output_cost_per_token: Final = rates.output_rate - - ## CALCULATE INPUT COST - prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token + priced: Final = require_priced( + price( + {"litellm_provider": "perplexity", **model_info}, + _native.UsageInput( + { + "input_cost_per_token": usage.prompt_tokens, + "output_cost_per_token": max(usage.completion_tokens - reasoning_tokens, 0), + "output_cost_per_reasoning_token": reasoning_tokens, + }, + prompt_tokens=usage.prompt_tokens, + ), + billing_context(current_time if current_time is not None else current_billing_time()), + ), + model, + ) + prompt_cost: float = priced.input + completion_cost: float = priced.output ## ADD CITATION TOKENS COST (if present) citation_tokens: Final = getattr(usage, "citation_tokens", 0) or 0 @@ -76,24 +87,6 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non citation_cost_per_token: Final = _safe_float_cast(citation_cost_value) prompt_cost += citation_tokens * citation_cost_per_token - ## CALCULATE OUTPUT COST - reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0 - if reasoning_tokens == 0 and hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details: - reasoning_tokens = getattr(usage.completion_tokens_details, "reasoning_tokens", 0) or 0 - - reasoning_cost_value: Final = model_info.get("output_cost_per_reasoning_token") - - # `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity usage - # convention (codified for the central path in PR #18607). When a reasoning rate is - # configured we subtract before the output-rate multiplication so the reasoning - # tokens are not billed twice. - if reasoning_tokens > 0 and reasoning_cost_value is not None: - non_reasoning_completion_tokens: Final = max(0, (usage.completion_tokens or 0) - reasoning_tokens) - completion_cost: float = non_reasoning_completion_tokens * output_cost_per_token - completion_cost += reasoning_tokens * _safe_float_cast(reasoning_cost_value) - else: - completion_cost = (usage.completion_tokens or 0) * output_cost_per_token - ## ADD SEARCH QUERIES COST (if present) num_search_queries = 0 if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details: diff --git a/litellm/llms/vertex_ai/cost_calculator.py b/litellm/llms/vertex_ai/cost_calculator.py index 864619f91b0..80cf1df176a 100644 --- a/litellm/llms/vertex_ai/cost_calculator.py +++ b/litellm/llms/vertex_ai/cost_calculator.py @@ -1,14 +1,18 @@ # What is this? ## Cost calculation for Google AI Studio / Vertex AI models +from collections.abc import Sequence from typing import Final, Literal import litellm from litellm import verbose_logger +from litellm._cost import billing_context, price, require_priced, token_usage +from litellm._internal_context import current_billing_time from litellm.litellm_core_utils.llm_cost_calc.utils import ( generic_cost_per_token, get_vertex_regional_endpoint_uplift, is_above_128k, ) +from litellm.rust_bridge import _native from litellm.types.utils import ModelInfo, Usage """ @@ -58,6 +62,30 @@ def cost_router( return "cost_per_character" +def _token_component_cost( + model: str, + model_info: ModelInfo, + usage: Usage, + direction: Literal["input", "output"], + service_tier: str | None, +) -> float: + if ( + model_info.get("input_cost_per_token_above_128k_tokens") is not None + or model_info.get("output_cost_per_token_above_128k_tokens") is not None + ): + input_cost, output_cost = _handle_128k_pricing(model_info, usage, direction=direction) + return input_cost if direction == "input" else output_cost + priced: Final = require_priced( + price( + model_info, + token_usage(usage, model_info, direction=direction), + billing_context(current_billing_time(), service_tier), + ), + model, + ) + return priced.input if direction == "input" else priced.output + + def cost_per_character( model: str, custom_llm_provider: str, @@ -91,12 +119,7 @@ def cost_per_character( ## CALCULATE INPUT COST if prompt_characters is None: - prompt_cost, _ = cost_per_token( - model=model, - custom_llm_provider=custom_llm_provider, - usage=usage, - service_tier=service_tier, - ) + prompt_cost = _token_component_cost(model, model_info, usage, "input", service_tier) else: try: if ( @@ -123,21 +146,11 @@ def cost_per_character( "litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - %s\nDefaulting to None", e, ) - prompt_cost, _ = cost_per_token( - model=model, - custom_llm_provider=custom_llm_provider, - usage=usage, - service_tier=service_tier, - ) + prompt_cost = _token_component_cost(model, model_info, usage, "input", service_tier) ## CALCULATE OUTPUT COST if completion_characters is None: - _, completion_cost = cost_per_token( - model=model, - custom_llm_provider=custom_llm_provider, - usage=usage, - service_tier=service_tier, - ) + completion_cost = _token_component_cost(model, model_info, usage, "output", service_tier) else: completion_tokens: Final = usage.completion_tokens try: @@ -164,12 +177,7 @@ def cost_per_character( "litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - %s\nDefaulting to None", e, ) - _, completion_cost = cost_per_token( - model=model, - custom_llm_provider=custom_llm_provider, - usage=usage, - service_tier=service_tier, - ) + completion_cost = _token_component_cost(model, model_info, usage, "output", service_tier) vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) return prompt_cost * vertex_uplift, completion_cost * vertex_uplift @@ -178,27 +186,27 @@ def cost_per_character( def _handle_128k_pricing( model_info: ModelInfo, usage: Usage, + *, + factors: Sequence[float] = (), + direction: Literal["input", "output"] | None = None, ) -> tuple[float, float]: - ## CALCULATE INPUT COST - input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens") - output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens") - - prompt_tokens: Final = usage.prompt_tokens - completion_tokens: Final = usage.completion_tokens - - if is_above_128k(tokens=prompt_tokens) and input_cost_per_token_above_128k_tokens is not None: - prompt_cost = prompt_tokens * input_cost_per_token_above_128k_tokens - else: - prompt_cost = prompt_tokens * (model_info["input_cost_per_token"] or 0.0) - - ## CALCULATE OUTPUT COST - output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens") - if is_above_128k(tokens=completion_tokens) and output_cost_per_token_above_128k_tokens is not None: - completion_cost = completion_tokens * output_cost_per_token_above_128k_tokens - else: - completion_cost = completion_tokens * (model_info["output_cost_per_token"] or 0.0) - - return prompt_cost, completion_cost + result: Final = require_priced( + price( + model_info, + _native.UsageInput( + { + **({"input_cost_per_token": usage.prompt_tokens} if direction != "output" else {}), + **({"output_cost_per_token": usage.completion_tokens} if direction != "input" else {}), + }, + prompt_tokens=usage.prompt_tokens, + ), + billing_context(current_billing_time()), + factors=factors, + directional_token_threshold=128000, + ), + model_info.get("key", "vertex_ai"), + ) + return result.input, result.output def cost_per_token( @@ -235,12 +243,11 @@ def cost_per_token( input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens") output_cost_per_token_above_128k_tokens: Final = model_info.get("output_cost_per_token_above_128k_tokens") if input_cost_per_token_above_128k_tokens is not None or output_cost_per_token_above_128k_tokens is not None: - prompt_cost_128k, completion_cost_128k = _handle_128k_pricing( + return _handle_128k_pricing( model_info=model_info, usage=usage, + factors=(get_vertex_regional_endpoint_uplift(model_info, vertex_location),), ) - vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) - return prompt_cost_128k * vertex_uplift, completion_cost_128k * vertex_uplift return generic_cost_per_token( model=model, diff --git a/litellm/main.py b/litellm/main.py index f30d93361e5..99c6d82ac12 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -155,7 +155,6 @@ from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( BaseUrlRoutingHint, - CustomPricingLiteLLMParams, ModelResponseStream, ProviderRoutingHints, RawRequestTypedDict, @@ -1115,7 +1114,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o ) from litellm.responses.bridge_check import ( bridge_reasoning_effort, - keep_registered_mode, str_keyed_mapping, ) from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site @@ -1232,93 +1230,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]: return values -def _build_custom_pricing_entry( - custom_llm_provider: str, - kwargs: dict, - model_info: dict | None = None, -) -> dict: - """Build a complete model cost entry from kwargs and model_info. - - Collects all CustomPricingLiteLLMParams fields present in kwargs and - merges metadata from model_info (mode, supports_prompt_caching, max_tokens) - so that register_model() receives the full pricing configuration. - """ - entry: Final[dict] = {"litellm_provider": custom_llm_provider} - - for field_name in CustomPricingLiteLLMParams.model_fields: - value = kwargs.get(field_name) - if value is not None: - entry[field_name] = value - - if model_info and isinstance(model_info, dict): - for key in ("mode", "supports_prompt_caching", "max_tokens"): - if key in model_info and model_info[key] is not None: - entry.setdefault(key, model_info[key]) - - return entry - - -def _get_router_deployment_id(kwargs: dict) -> str | None: - for metadata_key in ("litellm_metadata", "metadata"): - metadata = kwargs.get(metadata_key) or {} - if not isinstance(metadata, dict): - continue - deployment_model_info = metadata.get("model_info") or {} - if not isinstance(deployment_model_info, dict): - continue - deployment_id = deployment_model_info.get("id") - if deployment_id is not None: - return str(deployment_id) - return None - - -def _register_custom_pricing_for_request( - model: str, - custom_llm_provider: str, - kwargs: dict, - model_info: dict | None, -) -> None: - """Register per-request custom pricing in litellm.model_cost. - - Router-originated requests (identified by the deployment id the router puts - in metadata) get their full pricing registered under that unique id only; - the shared ``{provider}/{model}`` key receives the entry with pricing fields - stripped, mirroring Router._create_deployment. This keeps one deployment's - pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in - pricing used by sibling deployments of the same backend model. Direct SDK - calls keep the legacy behavior of registering the shared key with pricing. - Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on - deployment registration, since a call routes by its own ``model_info``. - """ - entry: Final = _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - shared_key: Final = f"{custom_llm_provider}/{model}" - deployment_id: Final = _get_router_deployment_id(kwargs) - if deployment_id is None: - litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False) - return - shared_entry: Final = keep_registered_mode( - shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry) - ) - litellm.register_model( - {deployment_id: entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - litellm.register_model( - {shared_key: shared_entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - - def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None: return ctx.metadata @@ -5750,19 +5661,6 @@ def completion( "aws_region_name", None ) # support region-based pricing for bedrock - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token: @@ -6705,12 +6603,6 @@ def embedding( headers.update(proxy_headers) except Exception as e: verbose_logger.warning("Failed to get proxy auth headers: %s", e) - ### CUSTOM MODEL COST ### - input_cost_per_token: Final = kwargs.get("input_cost_per_token", None) - output_cost_per_token: Final = kwargs.get("output_cost_per_token", None) - cost_per_second: Final = kwargs.get("cost_per_second", None) - input_cost_per_second: Final = kwargs.get("input_cost_per_second", None) - output_cost_per_second: Final = kwargs.get("output_cost_per_second", None) openai_params: Final = [ "user", "dimensions", @@ -6757,20 +6649,6 @@ def embedding( **non_default_params, ) - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=kwargs.get("model_info"), - ) - logging: Final[LiteLLMLoggingObj] = litellm_logging_obj logging.update_environment_variables( model=model, diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 79e385c376a..76c9c43e168 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -26,6 +26,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import NotRequired, ReadOnly, Required, TypedDict, Unpack import litellm +from litellm._catalog import is_priced from litellm._internal_context import with_service_target from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache, LimitedSizeOrderedDict @@ -605,7 +606,7 @@ def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None) # See: https://github.com/BerriAI/litellm/issues/24770 safe_name = str(model_name).replace("\n", "").replace("\r", "") served_deployments = _deployments_served_for(model_name, llm_router) - if not _is_cost_explicitly_configured(served_deployments): + if not (deployment_prices := tuple(_priced_deployment_infos(served_deployments))): verbose_proxy_logger.debug( "Model %s has zero cost but no explicit cost " "configuration in model_cost entry — treating as unknown " @@ -616,7 +617,11 @@ def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None) zero_cost_cache[model_name] = False return False - if _has_explicit_positive_cost(served_deployments): + if any( + _is_positive_cost(info.get("input_cost_per_token")) + or _is_positive_cost(info.get("output_cost_per_token")) + for info in deployment_prices + ): verbose_proxy_logger.debug( "Model %s routes to a deployment with an explicit positive per-token price (enforce budget)", safe_name, @@ -696,47 +701,12 @@ def _has_ptu_flat_cost(deployments: Sequence[DeploymentTypedDict]) -> bool: return False -def _is_cost_explicitly_configured(deployments: Sequence[DeploymentTypedDict]) -> bool: - """ - Check if any of the deployments has cost fields explicitly set in its - litellm.model_cost entry. - - When Router._create_deployment() registers a model not in the global - cost map, it creates a sparse entry like {"id": ""} with no cost - fields. _get_model_info_helper() then defaults missing costs to 0. - This function detects that scenario by checking the raw model_cost entry. - - It also reaches a deployment that prices itself through its ``model_info`` block, whose entry - lands in the cost map under the deployment id. - """ +def _priced_deployment_infos(deployments: Sequence[DeploymentTypedDict]) -> Iterator[Mapping[str, object]]: for deployment in deployments: - model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id") - if model_id is None: - continue - raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY) - if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry: - return True - return False - - -def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> bool: - """Whether any of the deployments carries an explicit positive per-token price. - - The group's price is read from the deployments named after the routed group and from - that group's own alias target, while an alias chain is served from the deployments named - after its routed group or from a wildcard route; a priced route among those serves the - request at its own rate whatever the group's price reads. - """ - for deployment in deployments: - model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id") - if model_id is None: - continue - raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY) - if _is_positive_cost(raw_entry.get("input_cost_per_token")) or _is_positive_cost( - raw_entry.get("output_cost_per_token") + if isinstance(model_id := (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id"), str) and is_priced( + model_id ): - return True - return False + yield litellm.get_model_info(model=model_id) _EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({}) @@ -798,7 +768,7 @@ def model_has_no_cost_mapping(model: str | None, llm_router: Router | None) -> b if _model_group_has_pricing(model=model, llm_router=llm_router): return False - return not _is_cost_explicitly_configured(llm_router.get_model_list(model_name=model) or ()) + return not any(_priced_deployment_infos(llm_router.get_model_list(model_name=model) or ())) def _unpriced_models_in_request(model: str | list[str] | None, llm_router: Router | None) -> tuple[str, ...]: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c64620fb29e..0c6ca3f16d1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -203,7 +203,7 @@ from litellm.types.utils import ( is_server_derived_pricing_key, pricing_override_fields, ) -from litellm.utils import cost_map_omits_token_price, load_credentials_from_list, with_named_credential +from litellm.utils import load_credentials_from_list, with_named_credential if TYPE_CHECKING: from aiohttp import ClientSession @@ -15996,7 +15996,11 @@ def _enrich_model_info_with_litellm_data( discovered_model_info: Final = ( llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({}) ) - unpriced: Final = cost_map_omits_token_price(model_info.get("id"), litellm_model_info.get("key")) + from litellm._catalog import is_priced + + unpriced: Final = not any( + is_priced(key) for key in (model_info.get("id"), litellm_model_info.get("key")) if isinstance(key, str) + ) stamped_model_info: Final = MappingProxyType( {**model_info, **_pricing_override_stamps(model_info, model.get("litellm_params") or MappingProxyType({}))} ) diff --git a/litellm/router.py b/litellm/router.py index 7993b5b921c..ad6a951a9de 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -52,6 +52,7 @@ from typing_extensions import assert_never, overload import litellm import litellm.litellm_core_utils.exception_mapping_utils from litellm import get_secret_str +from litellm._catalog import is_priced from litellm._internal_context import service_target, with_service_target from litellm._logging import verbose_router_logger from litellm._uuid import uuid @@ -11548,7 +11549,7 @@ class Router: A strategy-router alias is never the deployment actually called or billed, so custom pricing configured on it must not become a cost-map - price: an explicit zero would let ``_is_cost_explicitly_configured`` + price: an explicit zero would let the free-model budget policy treat the alias as a genuinely free model and waive budget checks for requests that route to (and bill as) a real deployment. """ @@ -12321,7 +12322,7 @@ class Router: 2. If not, check if litellm model name is in model info 3. If not, return None """ - from litellm.utils import cost_map_omits_token_price, update_dictionary + from litellm.utils import update_dictionary model_info: ModelInfo | None = None custom_model_info: dict | None = None @@ -12386,7 +12387,7 @@ class Router: builtin_key: Final = ( litellm_model_name_model_info.get("key") if litellm_model_name_model_info is not None else None ) - if cost_map_omits_token_price(model_id, builtin_key, base_model_key): + if not any(is_priced(key) for key in (model_id, builtin_key, base_model_key) if isinstance(key, str)): return cast( # cast-ok: TypedDict spread with overridden keys loses its type ModelInfo, {**model_info, "input_cost_per_token": None, "output_cost_per_token": None} ) diff --git a/litellm/utils.py b/litellm/utils.py index ee44513a371..6000bfdd8b5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3660,22 +3660,6 @@ def set_live_deployment_replay(replay: Callable[[], None]) -> None: _LiveDeploymentReplay.callbacks = (replay,) -def cost_map_omits_token_price(*keys: object) -> bool: - """Whether the raw ``litellm.model_cost`` entries under ``keys`` exist but none carries a per-token price. - - ``get_model_info`` substitutes 0 for a missing price, which reads exactly like a declared - zero. Surfaces that report pricing use this to keep an unpriced deployment at ``None``. - """ - entries: Final = tuple( - entry - for entry in (litellm.model_cost.get(key) for key in keys if isinstance(key, str)) - if isinstance(entry, dict) - ) - return len(entries) > 0 and not any( - "input_cost_per_token" in entry or "output_cost_per_token" in entry for entry in entries - ) - - def register_model( model_cost: str | dict, *, @@ -6322,25 +6306,8 @@ def get_model_info_helper( if _model_info is None or key is None: raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider)) - _input_cost_per_token: float | None = _model_info.get("input_cost_per_token") - if _input_cost_per_token is None: - # default value to 0, be noisy about this - verbose_logger.debug( - "model=%s, custom_llm_provider=%s has no input_cost_per_token in model_cost_map. Defaulting to 0.", - model, - custom_llm_provider, - ) - _input_cost_per_token = 0 - - _output_cost_per_token: float | None = _model_info.get("output_cost_per_token") - if _output_cost_per_token is None: - # default value to 0, be noisy about this - verbose_logger.debug( - "model=%s, custom_llm_provider=%s has no output_cost_per_token in model_cost_map. Defaulting to 0.", - model, - custom_llm_provider, - ) - _output_cost_per_token = 0 + _input_cost_per_token: Final = _model_info.get("input_cost_per_token") + _output_cost_per_token: Final = _model_info.get("output_cost_per_token") returned_model_info: Final = ModelInfoBase( key=key, diff --git a/tests/unit/_catalog/test_read.py b/tests/unit/_catalog/test_read.py index ce45c502b5b..74c916fd2ae 100644 --- a/tests/unit/_catalog/test_read.py +++ b/tests/unit/_catalog/test_read.py @@ -13,6 +13,90 @@ from litellm.rust_bridge._native import ModelNotFound, ResolvedModel from litellm.utils import ModelNotMappedError, _invalidate_model_cost_lowercase_map, get_model_cost_key +@pytest.mark.parametrize( + ("prices", "expected"), + ( + ({}, False), + ({"input_cost_per_token": None, "output_cost_per_token": None}, False), + ({"input_cost_per_token": 0, "output_cost_per_token": 0}, True), + ({"input_cost_per_token": 0.25}, True), + ({"output_cost_per_token": 0.5}, True), + ({"input_cost_per_token": -1}, False), + ({"input_cost_per_token": "invalid-price"}, False), + ({"input_cost_per_second": 1.0}, False), + ({"cache_read_input_token_cost": 0.0}, True), + ({"cache_creation_input_token_cost_above_1hr": 0.0}, True), + ({"output_cost_per_reasoning_token": 0.5}, True), + ({"input_cost_per_audio_token": 0.5}, True), + ({"output_cost_per_image_token": 0.5}, True), + ({"input_cost_per_token_priority": 0.5}, True), + ({"input_cost_per_token_batches": 0.5}, True), + ({"input_cost_per_token_above_128k_tokens": 0.5}, True), + ({"tiered_pricing": [{"range": [0, 200], "input_cost_per_token": 0.5}]}, True), + ({"tiered_pricing": [{"range": [100, 200], "input_cost_per_token": 0.5}]}, False), + ({"off_peak_pricing": {"hours_utc": "01:00-02:00", "input_cost_per_token": 0.5}}, True), + ( + { + "input_cost_per_token": 1.0, + "tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 2.0}], + }, + True, + ), + ), +) +def test_is_priced_distinguishes_explicit_token_rates( + monkeypatch: pytest.MonkeyPatch, prices: dict[str, object], expected: bool +) -> None: + from litellm._catalog import is_priced + + monkeypatch.setattr(litellm, "model_cost", {"openai/price-probe": {"litellm_provider": "openai", **prices}}) + assert is_priced("price-probe", "openai") is expected + assert is_priced("unmapped-price-probe", "openai") is False + + +def test_is_priced_tracks_selected_live_row_and_tombstones(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm._catalog import is_priced + + rows: Final[dict[str, dict[str, object]]] = { + "price-probe": {"litellm_provider": "openai", "input_cost_per_token": 1}, + "openai/price-probe": {"litellm_provider": "openai"}, + } + monkeypatch.setattr(litellm, "model_cost", rows) + assert is_priced("price-probe", "openai") is False + rows["openai/price-probe"]["input_cost_per_token"] = 0 + assert is_priced("price-probe", "openai") is True + rows["openai/price-probe"] = {"litellm_provider": "openai"} + assert is_priced("price-probe", "openai") is False + del rows["openai/price-probe"] + assert is_priced("price-probe", "openai") is True + del rows["price-probe"] + assert is_priced("price-probe", "openai") is False + + +def test_providerless_deployment_prices_without_writing_provider_context(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm._catalog import is_priced + + row: Final = {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0} + monkeypatch.setitem(litellm.model_cost, "providerless-priced-deployment", row) + assert is_priced("providerless-priced-deployment", "openai") is True + assert litellm.cost_per_token( + "providerless-priced-deployment", 10, 20, custom_llm_provider="openai" + ) == pytest.approx((10.0, 40.0)) + assert row == {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0} + + +@pytest.mark.parametrize("provider", (None, "", 123)) +def test_presence_does_not_repair_an_explicit_invalid_provider( + monkeypatch: pytest.MonkeyPatch, provider: object +) -> None: + from litellm._catalog import is_priced + + row: Final = {"litellm_provider": provider, "input_cost_per_token": 1.0, "output_cost_per_token": 2.0} + monkeypatch.setitem(litellm.model_cost, "openai/invalid-provider-price-probe", row) + assert is_priced("invalid-provider-price-probe", "openai") is False + assert row == {"litellm_provider": provider, "input_cost_per_token": 1.0, "output_cost_per_token": 2.0} + + def test_model_info_preserves_every_bundled_row() -> None: for key, row in litellm.model_cost.items(): result: Final = _read.resolve(key) diff --git a/tests/unit/_cost/test_price.py b/tests/unit/_cost/test_price.py new file mode 100644 index 00000000000..29ddce4dd76 --- /dev/null +++ b/tests/unit/_cost/test_price.py @@ -0,0 +1,309 @@ +from collections.abc import Mapping +from datetime import datetime, timezone +from typing import Final + +import pytest + +import litellm +from litellm._cost import billing_context, evaluate, price, require_priced, token_usage +from litellm._internal_context import pinned_billing_time +from litellm.exceptions import ModelNotMappedError +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.rust_bridge import _native +from litellm.types.utils import ModelResponse, Usage + + +@pytest.mark.parametrize( + ("tier_prices", "expected"), + ( + ({"output_cost_per_token": 4.0}, (10.0, 20.0)), + ({"input_cost_per_token": None, "output_cost_per_token": 4.0}, None), + ({"input_cost_per_token": 0.0, "output_cost_per_token": 4.0}, (0.0, 40.0)), + ({"input_cost_per_token": 3.0}, (30.0, 20.0)), + ({"input_cost_per_token": 3.0, "output_cost_per_token": None}, None), + ({"input_cost_per_token": 3.0, "output_cost_per_token": 0.0}, (30.0, 0.0)), + ), + ids=("input-absent", "input-null", "input-zero", "output-absent", "output-null", "output-zero"), +) +def test_tier_declared_null_is_unpriced_and_explicit_zero_remains_free( + monkeypatch: pytest.MonkeyPatch, tier_prices: Mapping[str, object], expected: tuple[float, float] | None +) -> None: + model: Final = "openai/m5-tier-presence" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "tiered_pricing": [{"range": [0, 100], **tier_prices}], + }, + ) + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=10)) + if expected is None: + with pytest.raises(ModelNotMappedError): + litellm.cost_per_token(model, 10, 10, custom_llm_provider="openai") + with pytest.raises(ModelNotMappedError): + litellm.completion_cost(response, model=model, custom_llm_provider="openai") + else: + assert litellm.cost_per_token(model, 10, 10, custom_llm_provider="openai") == pytest.approx(expected) + assert litellm.completion_cost(response, model=model, custom_llm_provider="openai") == pytest.approx( + sum(expected) + ) + + +@pytest.mark.parametrize( + ("rates", "usage", "expected"), + ( + ( + { + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 3.0, "output_cost_per_token": 4.0}], + }, + Usage(prompt_tokens=10, completion_tokens=10), + (30.0, 40.0), + ), + ( + { + "cache_read_input_token_cost": 9.0, + "tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 1.0, "output_cost_per_token": 2.0}], + }, + Usage(prompt_tokens=100, completion_tokens=10, prompt_tokens_details={"cached_tokens": 40}), + (100.0, 20.0), + ), + ( + { + "cache_creation_input_token_cost_above_1hr": 9.0, + "tiered_pricing": [ + { + "range": [0, 100], + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "cache_creation_input_token_cost": 3.0, + } + ], + }, + Usage( + prompt_tokens=100, + completion_tokens=10, + prompt_tokens_details={ + "cache_creation_tokens": 20, + "cache_creation_token_details": {"ephemeral_5m_input_tokens": 10, "ephemeral_1h_input_tokens": 10}, + }, + ), + (140.0, 20.0), + ), + ), + ids=("flat-and-tier", "tier-cache-read-fallback", "tier-hourly-cache-write-fallback"), +) +def test_public_token_pricing_preserves_tier_local_fallbacks( + monkeypatch: pytest.MonkeyPatch, rates: Mapping[str, object], usage: Usage, expected: tuple[float, float] +) -> None: + model: Final = "openai/m5-tier-fallback" + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **rates}) + assert litellm.cost_per_token( + model, usage.prompt_tokens, usage.completion_tokens, custom_llm_provider="openai", usage_object=usage + ) == pytest.approx(expected) + assert litellm.completion_cost( + ModelResponse(model=model, usage=usage), model=model, custom_llm_provider="openai" + ) == pytest.approx(sum(expected)) + + +@pytest.mark.parametrize("stream", (False, True)) +@pytest.mark.parametrize( + ("kind", "rates", "expected"), + ( + ("unknown", None, None), + ("metadata", {}, None), + ("free", {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, 0.0), + ("paid", {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}, 50.0), + ), +) +def test_unknown_free_and_priced_public_surfaces( + monkeypatch: pytest.MonkeyPatch, + stream: bool, + kind: str, + rates: Mapping[str, float] | None, + expected: float | None, +) -> None: + model: Final = f"openai/m5-{kind}" + if rates is not None: + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **rates}) + info: Final = litellm.get_model_info(model, custom_llm_provider="openai") + assert info["input_cost_per_token"] == rates.get("input_cost_per_token") + assert info["output_cost_per_token"] == rates.get("output_cost_per_token") + + usage: Final = Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + response: Final = ModelResponse(model=model, usage=usage) + logging: Final = Logging( + model=model, + messages=[{"role": "user", "content": "hello"}], + stream=stream, + call_type="completion", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="m5-price-surfaces", + function_id="m5-price-surfaces", + ) + logging.update_environment_variables(litellm_params={}, optional_params={}, custom_llm_provider="openai") + if expected is None: + with pytest.raises(ModelNotMappedError): + litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai", usage_object=usage) + with pytest.raises(ModelNotMappedError): + litellm.completion_cost(response, model=model, custom_llm_provider="openai") + assert logging.response_cost_calculator(response) is None + assert logging.model_call_details.get("response_cost_failure_debug_information") is not None + else: + split: Final = litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai", usage_object=usage) + assert split == pytest.approx((expected / 5, expected * 4 / 5)) + assert litellm.completion_cost(response, model=model, custom_llm_provider="openai") == pytest.approx(expected) + assert logging.response_cost_calculator(response) == pytest.approx(expected) + + +def test_public_pricing_reads_direct_row_updates(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "openai/m5-live-update" + row: Final = {"litellm_provider": "openai", "input_cost_per_token": 1.0, "output_cost_per_token": 2.0} + monkeypatch.setitem(litellm.model_cost, model, row) + assert litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai") == pytest.approx((10.0, 40.0)) + monkeypatch.setitem(row, "input_cost_per_token", 3.0) + assert litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai") == pytest.approx((30.0, 40.0)) + + +def test_invalid_price_is_not_authorized_by_permissive_model_info(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "openai/m5-invalid-row" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "input_cost_per_audio_token": "invalid", + }, + ) + assert litellm.get_model_info(model, custom_llm_provider="openai")["input_cost_per_token"] == 1.0 + with pytest.raises(ModelNotMappedError, match="input_cost_per_audio_token"): + litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai") + + +def test_custom_cost_uses_ephemeral_row_and_native_cache_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "openai/m5-custom-cost" + row: Final = {"litellm_provider": "openai", "input_cost_per_token": 10.0, "output_cost_per_token": 20.0} + monkeypatch.setitem(litellm.model_cost, model, row) + usage: Final = Usage( + prompt_tokens=100, + completion_tokens=10, + prompt_tokens_details={"cached_tokens": 40, "cache_creation_tokens": 20}, + ) + assert litellm.cost_per_token( + model, + 100, + 10, + usage_object=usage, + custom_cost_per_token={"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}, + ) == pytest.approx((100.0, 20.0)) + assert litellm.model_cost[model] is row + assert row["input_cost_per_token"] == 10.0 + assert litellm.cost_per_token(model, 100, 10, custom_llm_provider="openai") == pytest.approx((1000.0, 200.0)) + + +@pytest.mark.parametrize("tier", ("priority", "fast", " FAST ")) +def test_billing_context_normalizes_priority_alias(tier: str) -> None: + row: Final = {"litellm_provider": "openai", "input_cost_per_token": 1.0, "input_cost_per_token_priority": 3.0} + usage: Final = Usage(prompt_tokens=10, completion_tokens=0) + result: Final = require_priced( + price(row, token_usage(usage, row), billing_context(datetime(2026, 1, 1), tier)), + "priority", + ) + assert (result.input, result.output) == (30.0, 0.0) + + +def test_adapter_preserves_missing_meter_and_native_fallback_result() -> None: + when: Final = billing_context(datetime(2026, 1, 1)) + missing: Final = price({"litellm_provider": "openai"}, _native.UsageInput({"input_cost_per_token": 1}), when) + assert isinstance(missing, _native.Unpriced) + assert tuple(missing.missing) == ("input_cost_per_token",) + fallback: Final = require_priced( + price( + {"litellm_provider": "openai", "input_cost_per_token": 2.0}, + _native.UsageInput({"cache_read_input_token_cost": 10}), + when, + ), + "cache fallback", + ) + assert fallback.input == 20.0 + assert fallback.lines[0].fallback_from is not None + + +@pytest.mark.parametrize("factor", (1.0, 1.1)) +@pytest.mark.parametrize( + ("prompt", "completion", "cache", "tier", "expected", "rates"), + ( + (10, 10, 0, None, (10, 20), (1, 2)), + (128000, 128000, 0, None, (128000, 256000), (1, 2)), + (128001, 10, 0, None, (384003, 20), (3, 2)), + (10, 128001, 0, None, (10, 512004), (1, 4)), + (128001, 128001, 0, None, (384003, 512004), (3, 4)), + (100, 10, 80, None, (100, 20), (1, 2)), + (100, 10, 0, "priority", (100, 20), (1, 2)), + ), +) +def test_vertex_directional_route_uses_real_aggregate_quantities_and_native_projected_rates( + monkeypatch: pytest.MonkeyPatch, + factor: float, + prompt: int, + completion: int, + cache: int, + tier: str | None, + expected: tuple[float, float], + rates: tuple[float, float], +) -> None: + model: Final = "vertex_ai/gemini-2-directional-test" + row: Final = { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "input_cost_per_token_above_128k_tokens": 3.0, + "output_cost_per_token_above_128k_tokens": 4.0, + "cache_read_input_token_cost": 0.1, + "input_cost_per_token_priority": 5.0, + "output_cost_per_token_priority": 6.0, + "regional_endpoint_uplift_multiplier": factor, + "off_peak_pricing": {"hours_utc": "00:00-06:00", "input_cost_per_token": 0.1, "output_cost_per_token": 0.2}, + } + monkeypatch.setitem(litellm.model_cost, model, row) + shaped: Final = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") + usage: Final = Usage( + prompt_tokens=prompt, completion_tokens=completion, prompt_tokens_details={"cached_tokens": cache} + ) + when: Final = datetime(2026, 1, 1, 1, tzinfo=timezone.utc) + with pinned_billing_time(when): + actual: Final = litellm.cost_per_token( + model, + prompt, + completion, + custom_llm_provider="vertex_ai", + usage_object=usage, + service_tier=tier, + vertex_location="us-east5", + ) + assert actual == pytest.approx((expected[0] * factor, expected[1] * factor)) + for source in (row, shaped): + evaluated: Final = evaluate( + source, + _native.UsageInput( + {"input_cost_per_token": prompt, "output_cost_per_token": completion}, prompt_tokens=prompt + ), + billing_context(when, tier), + project=("input_cost_per_token", "output_cost_per_token"), + factors=(factor,), + directional_token_threshold=128000, + ) + assert isinstance(evaluated, _native.PriceEvaluation) + priced: Final = require_priced(evaluated.pricing, model) + assert (priced.input, priced.output) == pytest.approx(actual) + assert tuple(selected.unit_price for selected in evaluated.rates) == pytest.approx( + (rates[0] * factor, rates[1] * factor) + ) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 21b527250c4..9fb0037e73b 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1,4 +1,3 @@ -import json from collections.abc import Mapping from datetime import datetime, timezone from typing import Final, cast @@ -10,22 +9,11 @@ from litellm._internal_context import pinned_billing_time from litellm.litellm_core_utils.llm_cost_calc.utils import ( BilledTokenRates, CostCalculatorUtils, - PromptTokensDetailsResult, - TokenRates, - _calculate_input_cost, - _get_token_base_cost, - _is_off_peak, - _is_within_off_peak_window, - apply_off_peak_pricing, - apply_provider_cache_read_default, - calculate_cache_writing_cost, + calculate_prompt_caching_savings, generic_cost_per_token, get_billed_token_rates, get_token_type_cost_breakdown, ) -from litellm.llms.gemini.image_generation.cost_calculator import ( - cost_calculator as gemini_image_generation_cost_calculator, -) from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_image_generation_cost_calculator, ) @@ -42,6 +30,70 @@ from litellm.types.utils import ( ) +def _projected_rates( + model_info: ModelInfo, + usage: Usage, + service_tier: str | None = None, + current_time: datetime | None = None, +) -> BilledTokenRates: + rates: Final = get_billed_token_rates( + model="projection-fixture", + custom_llm_provider="custom", + usage=usage, + model_info=model_info, + service_tier=service_tier, + current_time=current_time, + ) + assert rates is not None + return rates + + +def _off_peak_cost(off_peak: Mapping[str, object], current_time: datetime) -> float: + prompt, _ = generic_cost_per_token( + "off-peak-fixture", + Usage(prompt_tokens=1), + "custom", + model_info={ + "input_cost_per_token": 10.0, + "output_cost_per_token": 0.0, + "off_peak_pricing": {**off_peak, "input_cost_per_token": 1.0}, + }, + current_time=current_time, + ) + return prompt + + +def _window_cost(window: str | list[str], current_time: datetime) -> float: + return _off_peak_cost({"hours_utc": window}, current_time) + + +def _cache_write_cost( + cache_creation_tokens: int, + cache_creation_token_details: CacheCreationTokenDetails | None, + cache_creation_cost: float | None, + cache_creation_cost_by_ttl: Mapping[str, float | None], +) -> float: + prompt, _ = generic_cost_per_token( + "cache-write-fixture", + Usage( + prompt_tokens=cache_creation_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + cache_creation_tokens=cache_creation_tokens, + cache_creation_token_details=cache_creation_token_details, + ), + ), + "custom", + model_info={ + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "cache_creation_input_token_cost": cache_creation_cost, + "cache_creation_input_token_cost_above_1hr": cache_creation_cost_by_ttl.get("1hr"), + "cache_creation_input_token_cost_above_12hr": cache_creation_cost_by_ttl.get("12hr"), + }, + ) + return prompt + + @pytest.fixture def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -61,12 +113,16 @@ def test_missing_cache_read_rate_resolves_to_input_rate(prompt_tokens, read_rate "cache_read_input_token_cost": read_rate, } usage = Usage(prompt_tokens=prompt_tokens, prompt_tokens_details={"cached_tokens": 100}) - billed = _get_token_base_cost(info, usage, service_tier=service_tier) + billed = _projected_rates(info, usage, service_tier=service_tier) prompt_cost, _ = generic_cost_per_token( "policy-fixture", usage, "openai", service_tier=service_tier, model_info=info ) - assert billed.cache_read_rate == pytest.approx(read_rate if read_rate is not None else billed.input_rate) - assert prompt_cost == pytest.approx((prompt_tokens - 100) * billed.input_rate + 100 * billed.cache_read_rate) + assert billed.cache_read_input_token_cost == pytest.approx( + read_rate if read_rate is not None else billed.input_cost_per_token + ) + assert prompt_cost == pytest.approx( + (prompt_tokens - 100) * billed.input_cost_per_token + 100 * billed.cache_read_input_token_cost + ) def test_generic_cost_per_token_bills_cache_reads_at_input_rate_when_no_cache_read_rate() -> None: @@ -124,17 +180,12 @@ def test_generic_cost_per_token_bills_a_character_counted_prompt_under_one_unit( assert (prompt_cost, completion_cost) == (pytest.approx(expected_prompt_cost), 0.0) -def test_apply_provider_cache_read_default_only_derives_a_rate_for_fireworks() -> None: - openai_info: ModelInfo = {"input_cost_per_token": 2e-6} - fireworks_info: ModelInfo = {"input_cost_per_token": 2e-6} - - assert apply_provider_cache_read_default(openai_info, "openai") is openai_info - assert apply_provider_cache_read_default(openai_info, None) is openai_info - - processed_fireworks_info = apply_provider_cache_read_default(fireworks_info, "fireworks_ai") - - assert processed_fireworks_info is not fireworks_info - assert processed_fireworks_info["cache_read_input_token_cost"] == pytest.approx(2e-6 * 0.5) +@pytest.mark.parametrize(("provider", "expected"), (("openai", 0.0002), ("fireworks_ai", 0.0001))) +def test_only_fireworks_receives_a_default_cache_read_discount(provider: str, expected: float) -> None: + row: Final = {"input_cost_per_token": 2e-6} + usage: Final = Usage(prompt_tokens=100, completion_tokens=0, prompt_tokens_details={"cached_tokens": 100}) + assert generic_cost_per_token("cache-default", usage, provider, model_info=row) == pytest.approx((expected, 0)) + assert row == {"input_cost_per_token": 2e-6} def test_generic_cost_per_token_prefers_audio_per_second_rate() -> None: @@ -247,8 +298,8 @@ def test_missing_cache_read_uses_off_peak_input_rate(): "off_peak_pricing": {"hours_utc": "00:00-23:59", "input_cost_per_token": 5e-6}, } when = datetime(2026, 9, 7, 12, tzinfo=timezone.utc) - billed = _get_token_base_cost(info, Usage(prompt_tokens=100), current_time=when) - assert billed.input_rate == billed.cache_read_rate == 5e-6 + billed = _projected_rates(info, Usage(prompt_tokens=100), current_time=when) + assert billed.input_cost_per_token == billed.cache_read_input_token_cost == 5e-6 def test_reasoning_tokens_no_price_set(_local_model_cost_map): @@ -561,7 +612,7 @@ def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map): ) -def test_get_token_base_cost_picks_highest_crossed_tier(): +def test_billed_token_rates_picks_highest_crossed_tier(): """Regression test for #30345. With graduated tiers at 90k and 128k whose keys have different digit lengths, a request @@ -576,33 +627,33 @@ def test_get_token_base_cost_picks_highest_crossed_tier(): } usage = Usage(prompt_tokens=150_000, completion_tokens=10, total_tokens=150_010) - prompt_base_cost = _get_token_base_cost(model_info, usage).input_rate + prompt_base_cost = _projected_rates(model_info, usage).input_cost_per_token assert prompt_base_cost == 9e-6 -def test_is_within_off_peak_window_same_day(): +def test_window_cost_same_day(): from datetime import datetime, timezone window = "09:00-17:00" - assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) is True - assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 8, 59, tzinfo=timezone.utc)) is False - assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 9, 0, tzinfo=timezone.utc)) is True - assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 17, 0, tzinfo=timezone.utc)) is False + assert _window_cost(window, datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) == 1.0 + assert _window_cost(window, datetime(2026, 1, 1, 8, 59, tzinfo=timezone.utc)) == 10.0 + assert _window_cost(window, datetime(2026, 1, 1, 9, 0, tzinfo=timezone.utc)) == 1.0 + assert _window_cost(window, datetime(2026, 1, 1, 17, 0, tzinfo=timezone.utc)) == 10.0 -def test_is_within_off_peak_window_wraps_midnight(): +def test_window_cost_wraps_midnight(): from datetime import datetime, timezone window = "16:30-00:30" - assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) is True - assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 0, 15, tzinfo=timezone.utc)) is True - assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 16, 30, tzinfo=timezone.utc)) is True - assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 0, 30, tzinfo=timezone.utc)) is False - assert _is_within_off_peak_window(window, datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) is False + assert _window_cost(window, datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) == 1.0 + assert _window_cost(window, datetime(2026, 1, 1, 0, 15, tzinfo=timezone.utc)) == 1.0 + assert _window_cost(window, datetime(2026, 1, 1, 16, 30, tzinfo=timezone.utc)) == 1.0 + assert _window_cost(window, datetime(2026, 1, 1, 0, 30, tzinfo=timezone.utc)) == 10.0 + assert _window_cost(window, datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) == 10.0 -def test_is_within_off_peak_window_equal_start_and_end_covers_whole_day(): +def test_window_cost_equal_start_and_end_covers_whole_day(): """An equal start and end is the natural way to spell off-peak all day. It used to take the non-wrap branch, where start <= now < end can never hold, so it matched nothing and billed at standard rates around the clock without raising or logging anything.""" @@ -610,47 +661,45 @@ def test_is_within_off_peak_window_equal_start_and_end_covers_whole_day(): for window in ("00:00-00:00", "10:00-10:00"): for hour in range(24): - assert _is_within_off_peak_window(window, datetime(2026, 1, 1, hour, 0, tzinfo=timezone.utc)) is True, ( + assert _window_cost(window, datetime(2026, 1, 1, hour, 0, tzinfo=timezone.utc)) == 1.0, ( f"{window} should cover {hour:02d}:00" ) -def test_is_within_off_peak_window_multiple_windows(): +def test_window_cost_multiple_windows(): from datetime import datetime, timezone # Providers like DeepSeek V4 have more than one daily peak/off-peak window. windows = ["01:00-05:00", "13:00-16:00"] - assert _is_within_off_peak_window(windows, datetime(2026, 1, 1, 3, 0, tzinfo=timezone.utc)) is True - assert _is_within_off_peak_window(windows, datetime(2026, 1, 1, 14, 30, tzinfo=timezone.utc)) is True - assert _is_within_off_peak_window(windows, datetime(2026, 1, 1, 9, 0, tzinfo=timezone.utc)) is False - # a malformed entry in the list is ignored, valid entries still match - assert _is_within_off_peak_window(["bad", "13:00-16:00"], datetime(2026, 1, 1, 14, 0, tzinfo=timezone.utc)) is True - assert _is_within_off_peak_window([], datetime(2026, 1, 1, 14, 0, tzinfo=timezone.utc)) is False + assert _window_cost(windows, datetime(2026, 1, 1, 3, 0, tzinfo=timezone.utc)) == 1.0 + assert _window_cost(windows, datetime(2026, 1, 1, 14, 30, tzinfo=timezone.utc)) == 1.0 + assert _window_cost(windows, datetime(2026, 1, 1, 9, 0, tzinfo=timezone.utc)) == 10.0 + with pytest.raises(litellm.ModelNotMappedError, match="off_peak_pricing"): + _window_cost(["bad", "13:00-16:00"], datetime(2026, 1, 1, 14, 0, tzinfo=timezone.utc)) + with pytest.raises(litellm.ModelNotMappedError, match="off_peak_pricing"): + _window_cost([], datetime(2026, 1, 1, 14, 0, tzinfo=timezone.utc)) -def test_is_within_off_peak_window_normalizes_timezone_aware_input(): +def test_window_cost_normalizes_timezone_aware_input(): from datetime import datetime, timedelta, timezone # A caller may pass a non-UTC aware datetime; the window is UTC and must be # evaluated in UTC, not against the caller's wall-clock. 09:00 at UTC+8 is # 01:00 UTC, inside the 01:00-05:00 window. tz_plus_8 = timezone(timedelta(hours=8)) - assert _is_within_off_peak_window("01:00-05:00", datetime(2026, 1, 1, 9, 0, tzinfo=tz_plus_8)) is True - assert _is_within_off_peak_window("01:00-05:00", datetime(2026, 1, 1, 12, 0, tzinfo=tz_plus_8)) is True + assert _window_cost("01:00-05:00", datetime(2026, 1, 1, 9, 0, tzinfo=tz_plus_8)) == 1.0 + assert _window_cost("01:00-05:00", datetime(2026, 1, 1, 12, 0, tzinfo=tz_plus_8)) == 1.0 # 06:00 at UTC+8 is 22:00 UTC the previous day, outside the window - assert _is_within_off_peak_window("01:00-05:00", datetime(2026, 1, 1, 6, 0, tzinfo=tz_plus_8)) is False + assert _window_cost("01:00-05:00", datetime(2026, 1, 1, 6, 0, tzinfo=tz_plus_8)) == 10.0 -def test_is_within_off_peak_window_malformed_returns_false(): - from datetime import datetime, timezone - - now = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) - assert _is_within_off_peak_window("not-a-window", now) is False - assert _is_within_off_peak_window("16:30", now) is False - assert _is_within_off_peak_window("25:00-26:00", now) is False +@pytest.mark.parametrize("window", ("not-a-window", "16:30", "25:00-26:00")) +def test_malformed_window_is_unknown_pricing(window: str) -> None: + with pytest.raises(litellm.ModelNotMappedError, match="off_peak_pricing"): + _window_cost(window, datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) -def test_is_off_peak_weekday_qualified_windows_deepseek_schedule(): +def test_off_peak_cost_weekday_qualified_windows_deepseek_schedule(): """DeepSeek since 2026-08-23: peak is 01:00-04:00 and 06:00-10:00 UTC on weekdays only, with weekends off-peak around the clock. The weekday axis is not a filter on one window set; on two days of seven the off-peak window becomes the whole day, so the schedule needs two @@ -678,12 +727,12 @@ def test_is_off_peak_weekday_qualified_windows_deepseek_schedule(): datetime(2026, 8, 24, 0, 30, tzinfo=timezone.utc), ] for when in peak_instants: - assert _is_off_peak(deepseek, when) is False, f"{when.isoformat()} should bill peak" + assert _off_peak_cost(deepseek, when) == 10.0, f"{when.isoformat()} should bill peak" for when in off_peak_instants: - assert _is_off_peak(deepseek, when) is True, f"{when.isoformat()} should bill off-peak" + assert _off_peak_cost(deepseek, when) == 1.0, f"{when.isoformat()} should bill off-peak" -def test_is_off_peak_weekday_timezone_reads_vendor_calendar(): +def test_off_peak_cost_weekday_timezone_reads_vendor_calendar(): """The UTC and Asia/Shanghai calendars only disagree about the date over 16:00-24:00 UTC, so a window in that stretch is the one place a vendor-local weekday differs from a UTC one: 2026-08-28T16:30Z is Friday in UTC but already Saturday in Beijing.""" @@ -693,65 +742,69 @@ def test_is_off_peak_weekday_timezone_reads_vendor_calendar(): "weekday_timezone": "Asia/Shanghai", "windows": [{"hours_utc": "16:00-17:00", "weekdays": [6]}], } - assert _is_off_peak(shanghai_saturday, datetime(2026, 8, 28, 16, 30, tzinfo=timezone.utc)) is True - assert _is_off_peak(shanghai_saturday, datetime(2026, 8, 29, 16, 30, tzinfo=timezone.utc)) is False + assert _off_peak_cost(shanghai_saturday, datetime(2026, 8, 28, 16, 30, tzinfo=timezone.utc)) == 1.0 + assert _off_peak_cost(shanghai_saturday, datetime(2026, 8, 29, 16, 30, tzinfo=timezone.utc)) == 10.0 -def test_is_off_peak_weekdays_default_utc_calendar_and_accept_names(): +def test_off_peak_cost_weekdays_default_utc_calendar_and_accept_names(): from datetime import datetime, timezone named_weekend = {"windows": [{"hours_utc": "00:00-00:00", "weekdays": ["Sat", "sunday"]}]} - assert _is_off_peak(named_weekend, datetime(2026, 8, 29, 12, 0, tzinfo=timezone.utc)) is True - assert _is_off_peak(named_weekend, datetime(2026, 8, 28, 12, 0, tzinfo=timezone.utc)) is False + assert _off_peak_cost(named_weekend, datetime(2026, 8, 29, 12, 0, tzinfo=timezone.utc)) == 1.0 + assert _off_peak_cost(named_weekend, datetime(2026, 8, 28, 12, 0, tzinfo=timezone.utc)) == 10.0 utc_friday = {"windows": [{"hours_utc": "16:00-17:00", "weekdays": [5]}]} - assert _is_off_peak(utc_friday, datetime(2026, 8, 28, 16, 30, tzinfo=timezone.utc)) is True - assert _is_off_peak(utc_friday, datetime(2026, 8, 29, 16, 30, tzinfo=timezone.utc)) is False + assert _off_peak_cost(utc_friday, datetime(2026, 8, 28, 16, 30, tzinfo=timezone.utc)) == 1.0 + assert _off_peak_cost(utc_friday, datetime(2026, 8, 29, 16, 30, tzinfo=timezone.utc)) == 10.0 -def test_is_off_peak_naive_current_time_read_as_utc(): +def test_off_peak_cost_naive_current_time_read_as_utc(): from datetime import datetime block = {"windows": [{"hours_utc": "16:00-17:00", "weekdays": [5]}]} - assert _is_off_peak(block, datetime(2026, 8, 28, 16, 30)) is True - assert _is_off_peak(block, datetime(2026, 8, 29, 16, 30)) is False + assert _off_peak_cost(block, datetime(2026, 8, 28, 16, 30)) == 1.0 + assert _off_peak_cost(block, datetime(2026, 8, 29, 16, 30)) == 10.0 -def test_is_off_peak_invalid_weekday_timezone_falls_back_to_utc(): +def test_off_peak_cost_invalid_weekday_timezone_falls_back_to_utc(): from datetime import datetime, timezone block = {"weekday_timezone": "Not/AZone", "windows": [{"hours_utc": "16:00-17:00", "weekdays": [5]}]} - assert _is_off_peak(block, datetime(2026, 8, 28, 16, 30, tzinfo=timezone.utc)) is True - assert _is_off_peak(block, datetime(2026, 8, 29, 16, 30, tzinfo=timezone.utc)) is False + assert _off_peak_cost(block, datetime(2026, 8, 28, 16, 30, tzinfo=timezone.utc)) == 1.0 + assert _off_peak_cost(block, datetime(2026, 8, 29, 16, 30, tzinfo=timezone.utc)) == 10.0 -def test_is_off_peak_ignores_malformed_weekday_rules(): - from datetime import datetime, timezone - - when = datetime(2026, 8, 29, 12, 0, tzinfo=timezone.utc) - assert _is_off_peak({"windows": [{"hours_utc": "00:00-00:00", "weekdays": []}]}, when) is False - assert _is_off_peak({"windows": [{"hours_utc": "00:00-00:00", "weekdays": [0, 8, "noday", True]}]}, when) is False - assert _is_off_peak({"windows": [{"weekdays": [6]}]}, when) is False - assert _is_off_peak({"windows": [{"hours_utc": 1630}]}, when) is False - assert _is_off_peak({"windows": ["00:00-00:00"]}, when) is False - assert _is_off_peak({"windows": "00:00-00:00"}, when) is False - assert _is_off_peak({"hours_utc": 1630}, when) is False - assert _is_off_peak({}, when) is False +@pytest.mark.parametrize( + "block", + ( + {"windows": [{"hours_utc": "00:00-00:00", "weekdays": []}]}, + {"windows": [{"hours_utc": "00:00-00:00", "weekdays": [0, 8, "noday", True]}]}, + {"windows": [{"weekdays": [6]}]}, + {"windows": [{"hours_utc": 1630}]}, + {"windows": ["00:00-00:00"]}, + {"windows": "00:00-00:00"}, + {"hours_utc": 1630}, + {}, + ), +) +def test_malformed_weekday_rules_are_unknown_pricing(block: Mapping[str, object]) -> None: + with pytest.raises(litellm.ModelNotMappedError, match="off_peak_pricing"): + _off_peak_cost(block, datetime(2026, 8, 29, 12, 0, tzinfo=timezone.utc)) -def test_is_off_peak_flat_hours_and_windows_are_a_union(): +def test_off_peak_cost_flat_hours_and_windows_are_a_union(): from datetime import datetime, timezone block = { "hours_utc": "04:00-06:00", "windows": [{"hours_utc": "00:00-00:00", "weekdays": [7]}], } - assert _is_off_peak(block, datetime(2026, 8, 28, 5, 0, tzinfo=timezone.utc)) is True - assert _is_off_peak(block, datetime(2026, 8, 30, 20, 0, tzinfo=timezone.utc)) is True - assert _is_off_peak(block, datetime(2026, 8, 28, 20, 0, tzinfo=timezone.utc)) is False + assert _off_peak_cost(block, datetime(2026, 8, 28, 5, 0, tzinfo=timezone.utc)) == 1.0 + assert _off_peak_cost(block, datetime(2026, 8, 30, 20, 0, tzinfo=timezone.utc)) == 1.0 + assert _off_peak_cost(block, datetime(2026, 8, 28, 20, 0, tzinfo=timezone.utc)) == 10.0 -def test_get_token_base_cost_weekend_only_off_peak_rate(): +def test_billed_token_rates_weekend_only_off_peak_rate(): from datetime import datetime, timezone from typing import cast @@ -774,18 +827,18 @@ def test_get_token_base_cost_weekend_only_off_peak_rate(): ) usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - saturday_peak_hours = _get_token_base_cost( + saturday_peak_hours = _projected_rates( model_info, usage, current_time=datetime(2026, 8, 29, 2, 0, tzinfo=timezone.utc) ) - assert (saturday_peak_hours.input_rate, saturday_peak_hours.output_rate) == (5e-7, 1e-6) + assert (saturday_peak_hours.input_cost_per_token, saturday_peak_hours.output_cost_per_token) == (5e-7, 1e-6) - monday_same_hours = _get_token_base_cost( + monday_same_hours = _projected_rates( model_info, usage, current_time=datetime(2026, 8, 24, 2, 0, tzinfo=timezone.utc) ) - assert (monday_same_hours.input_rate, monday_same_hours.output_rate) == (1e-6, 2e-6) + assert (monday_same_hours.input_cost_per_token, monday_same_hours.output_cost_per_token) == (1e-6, 2e-6) -def test_get_token_base_cost_applies_off_peak_pricing(): +def test_billed_token_rates_applies_off_peak_pricing(): from datetime import datetime, timezone from typing import cast @@ -807,22 +860,18 @@ def test_get_token_base_cost_applies_off_peak_pricing(): ) usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert off_peak.input_rate == 5e-7 - assert off_peak.output_rate == 1e-6 - assert off_peak.cache_read_rate == 5e-8 + off_peak = _projected_rates(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) + assert off_peak.input_cost_per_token == 5e-7 + assert off_peak.output_cost_per_token == 1e-6 + assert off_peak.cache_read_input_token_cost == 5e-8 - peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert peak.input_rate == 1e-6 - assert peak.output_rate == 2e-6 - assert peak.cache_read_rate == 1e-7 + peak = _projected_rates(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) + assert peak.input_cost_per_token == 1e-6 + assert peak.output_cost_per_token == 2e-6 + assert peak.cache_read_input_token_cost == 1e-7 -def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates(): - """A truthy non-mapping off_peak_pricing value (a bare string or a list in - YAML) must bill standard rates rather than raising, matching how every - other malformed piece of the block behaves. - """ +def test_billed_token_rates_non_mapping_off_peak_block_is_unknown(): from datetime import datetime, timezone from typing import cast @@ -840,12 +889,21 @@ def test_get_token_base_cost_non_mapping_off_peak_block_bills_standard_rates(): "off_peak_pricing": malformed_block, }, ) - result = _get_token_base_cost(model_info, usage, current_time=when) - assert result.input_rate == 1e-6 - assert result.output_rate == 2e-6 + assert ( + get_billed_token_rates( + model="invalid-off-peak", + custom_llm_provider="custom", + usage=usage, + model_info=model_info, + current_time=when, + ) + is None + ) + with pytest.raises(litellm.ModelNotMappedError, match="off_peak_pricing"): + generic_cost_per_token("invalid-off-peak", usage, "custom", model_info=model_info, current_time=when) -def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset(): +def test_billed_token_rates_off_peak_falls_back_to_standard_when_unset(): from datetime import datetime, timezone from typing import cast @@ -861,12 +919,12 @@ def test_get_token_base_cost_off_peak_falls_back_to_standard_when_unset(): ) usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - result = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert result.input_rate == 5e-7 - assert result.output_rate == 2e-6 + result = _projected_rates(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) + assert result.input_cost_per_token == 5e-7 + assert result.output_cost_per_token == 2e-6 -def test_get_token_base_cost_off_peak_wins_over_threshold(): +def test_billed_token_rates_off_peak_wins_over_threshold(): from datetime import datetime, timezone from typing import cast @@ -888,13 +946,13 @@ def test_get_token_base_cost_off_peak_wins_over_threshold(): ) usage = Usage(prompt_tokens=250000, completion_tokens=250000, total_tokens=500000) - off_peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert off_peak.input_rate == 5e-7 - assert off_peak.output_rate == 1e-6 + off_peak = _projected_rates(model_info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) + assert off_peak.input_cost_per_token == 5e-7 + assert off_peak.output_cost_per_token == 1e-6 - peak = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert peak.input_rate == 3e-6 - assert peak.output_rate == 4e-6 + peak = _projected_rates(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) + assert peak.input_cost_per_token == 3e-6 + assert peak.output_cost_per_token == 4e-6 def test_get_model_info_propagates_off_peak_fields(): @@ -920,7 +978,7 @@ def test_get_model_info_propagates_off_peak_fields(): assert info["off_peak_pricing"] == off_peak_pricing -def test_get_token_base_cost_off_peak_wins_over_tiered_pricing(): +def test_billed_token_rates_off_peak_wins_over_tiered_pricing(): """Tiered pricing resolves base rates on its own path and returns early, so off-peak has to be applied there too or a model carrying both would silently bill the tier rate all day.""" from datetime import datetime, timezone @@ -945,11 +1003,11 @@ def test_get_token_base_cost_off_peak_wins_over_tiered_pricing(): info = litellm.get_model_info(model=model_name) usage = Usage(prompt_tokens=1_000, completion_tokens=100, total_tokens=1_100) - inside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) - assert (inside.input_rate, inside.output_rate) == (5e-7, 1e-6) + inside = _projected_rates(info, usage, current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc)) + assert (inside.input_cost_per_token, inside.output_cost_per_token) == (5e-7, 1e-6) - outside = _get_token_base_cost(info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert (outside.input_rate, outside.output_rate) == (3e-6, 6e-6) + outside = _projected_rates(info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) + assert (outside.input_cost_per_token, outside.output_cost_per_token) == (3e-6, 6e-6) def _register_off_peak_reasoning_model( @@ -1106,32 +1164,29 @@ def test_generic_cost_per_token_off_peak_reasoning_rate_wins_over_the_service_ti assert outside == pytest.approx(50 * 3e-6 + 30 * 6e-6) -def test_apply_off_peak_pricing_treats_bool_as_unset_and_parses_strings(): - from datetime import datetime, timezone +def test_boolean_off_peak_price_is_unknown_instead_of_unset() -> None: + with pytest.raises(litellm.ModelNotMappedError, match="cache_creation_input_token_cost"): + _off_peak_cost( + {"hours_utc": "16:30-00:30", "cache_creation_input_token_cost": True}, + datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc), + ) - model_name = "litellm-test-off-peak-odd-values" - _register_off_peak_reasoning_model( - model_name, + +def test_off_peak_reasoning_price_accepts_numeric_strings() -> None: + rates: Final = _projected_rates( { - "hours_utc": "16:30-00:30", - "cache_creation_input_token_cost": True, - "output_cost_per_reasoning_token": "5e-7", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "output_cost_per_reasoning_token": 4e-6, + "off_peak_pricing": {"hours_utc": "16:30-00:30", "output_cost_per_reasoning_token": "5e-7"}, }, + Usage(prompt_tokens=100, completion_tokens=50), + current_time=datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc), ) - standard = TokenRates( - input_rate=1e-6, output_rate=2e-6, cache_read_rate=1e-7, cache_creation_rate=1.25e-6, reasoning_rate=4e-6 - ) - - rates = apply_off_peak_pricing( - litellm.get_model_info(model_name, custom_llm_provider="openai"), - datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc), - standard, - ) - assert rates.cache_creation_rate == 1.25e-6 - assert rates.reasoning_rate == 5e-7 + assert rates.output_cost_per_reasoning_token == 5e-7 -def test_get_token_base_cost_off_peak_cache_creation_rate(): +def test_billed_token_rates_off_peak_cache_creation_rate(): from datetime import datetime, timezone from typing import cast @@ -1150,18 +1205,18 @@ def test_get_token_base_cost_off_peak_cache_creation_rate(): usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) inside_window = datetime(2026, 1, 1, 18, 0, tzinfo=timezone.utc) - inside = _get_token_base_cost(model_info, usage, current_time=inside_window) - assert inside.cache_creation_rate == 5e-7 - assert inside.cache_creation_rates_by_ttl["1hr"] == 2e-6 + inside = _projected_rates(model_info, usage, current_time=inside_window) + assert inside.cache_creation_input_token_cost == 5e-7 + assert inside.cache_creation_input_token_cost_by_ttl["1hr"] == 2e-6 - outside = _get_token_base_cost(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - assert outside.cache_creation_rate == 1.25e-6 + outside = _projected_rates(model_info, usage, current_time=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) + assert outside.cache_creation_input_token_cost == 1.25e-6 without_key = cast( ModelInfo, {**model_info, "off_peak_pricing": {"hours_utc": "16:30-00:30", "input_cost_per_token": 5e-7}}, ) - assert _get_token_base_cost(without_key, usage, current_time=inside_window).cache_creation_rate == 1.25e-6 + assert _projected_rates(without_key, usage, current_time=inside_window).cache_creation_input_token_cost == 1.25e-6 def test_get_token_type_cost_breakdown_reflects_off_peak_reasoning_and_cache_creation_rates(): @@ -1918,10 +1973,8 @@ def test_calculate_cost_component_with_string_values(): assert cost == 0.0 -def test_string_cost_values_edge_cases(): +def test_invalid_string_token_rate_is_unknown_instead_of_free(): """Test edge cases for string cost value handling.""" - from unittest.mock import patch - # Test with mixed string and float cost values mock_model_info = { "input_cost_per_token": "1e-6", # String @@ -1946,23 +1999,11 @@ def test_string_cost_values_edge_cases(): ), ) - with patch( - "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", - return_value=mock_model_info, - ): - prompt_cost, completion_cost = generic_cost_per_token( - model="test-model", usage=usage, custom_llm_provider="test-provider" + with pytest.raises(litellm.ModelNotMappedError, match="input_cost_per_audio_token"): + generic_cost_per_token( + model="test-model", usage=usage, custom_llm_provider="test-provider", model_info=mock_model_info ) - # Expected costs: - # Prompt: 1000 * 1e-6 + 100 * 0 (invalid string becomes 0) - # Completion: 500 * 2e-6 + 50 * 2e-6 (audio tokens fall back to base cost when output_cost_per_audio_token is None) - expected_prompt_cost = 1000 * 1e-6 - expected_completion_cost = 500 * 2e-6 + 50 * 2e-6 - - assert round(prompt_cost, 12) == round(expected_prompt_cost, 12) - assert round(completion_cost, 12) == round(expected_completion_cost, 12) - def test_string_cost_values_with_threshold(): """Test that string cost values work correctly with threshold pricing.""" @@ -1999,7 +2040,7 @@ def test_string_cost_values_with_threshold(): assert round(completion_cost, 12) == round(expected_completion_cost, 12) -def test_calculate_cache_writing_cost(): +def test__cache_write_cost(): """Test the calculate_cache_writing_cost function with detailed cache creation token breakdown.""" # Test case 1: With cache creation token details (matching the provided input) @@ -2010,7 +2051,7 @@ def test_calculate_cache_writing_cost(): cache_creation_cost_above_1hr = 6e-06 cache_creation_cost = 3.75e-06 - result = calculate_cache_writing_cost( + result = _cache_write_cost( cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, cache_creation_cost=cache_creation_cost, @@ -2031,7 +2072,7 @@ def test_calculate_cache_writing_cost(): cache_creation_token_details_none = None cache_creation_cost_fallback = 5e-06 - result_no_details = calculate_cache_writing_cost( + result_no_details = _cache_write_cost( cache_creation_tokens=cache_creation_tokens_no_details, cache_creation_token_details=cache_creation_token_details_none, cache_creation_cost=cache_creation_cost_fallback, @@ -2049,7 +2090,7 @@ def test_calculate_cache_writing_cost(): ephemeral_5m_input_tokens=None, ephemeral_1h_input_tokens=100 ) - result_partial = calculate_cache_writing_cost( + result_partial = _cache_write_cost( cache_creation_tokens=500, cache_creation_token_details=cache_creation_token_details_partial, cache_creation_cost=3e-06, @@ -2061,7 +2102,7 @@ def test_calculate_cache_writing_cost(): assert round(result_partial, 6) == round(expected_cost_partial, 6) # Test case 4: Zero costs - result_zero = calculate_cache_writing_cost( + result_zero = _cache_write_cost( cache_creation_tokens=1000, cache_creation_token_details=CacheCreationTokenDetails( ephemeral_5m_input_tokens=50, ephemeral_1h_input_tokens=950 @@ -2075,7 +2116,7 @@ def test_calculate_cache_writing_cost(): @pytest.mark.parametrize("field", CacheCreationTokenDetails.model_fields) def test_calculate_cache_writing_cost_bills_every_reported_lifetime(field: str) -> None: - cost = calculate_cache_writing_cost( + cost = _cache_write_cost( cache_creation_tokens=10, cache_creation_token_details=CacheCreationTokenDetails(**{field: 10}), cache_creation_cost=2.0, @@ -2085,7 +2126,7 @@ def test_calculate_cache_writing_cost_bills_every_reported_lifetime(field: str) def test_calculate_cache_writing_cost_bills_writes_of_an_unknown_lifetime_at_the_base_rate() -> None: - cost = calculate_cache_writing_cost( + cost = _cache_write_cost( cache_creation_tokens=10, cache_creation_token_details=CacheCreationTokenDetails( ephemeral_30m_input_tokens=2, ephemeral_12h_input_tokens=3 @@ -2222,49 +2263,16 @@ def test_generic_cost_per_token_prices_ultrafast_12h_cache_writes_at_the_ultrafa def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): - """ - Regression test: when cache_creation_tokens is 0 but cache_creation_token_details - has non-zero ephemeral tokens, the cost must still be calculated. - This ensures the guard in _calculate_input_cost doesn't skip - calculate_cache_writing_cost when only ephemeral token details are present. - """ - cache_creation_cost = 3.75e-06 - cache_creation_cost_above_1hr = 6e-06 - - prompt_tokens_details: PromptTokensDetailsResult = { - "cache_hit_tokens": 0, - "cache_hit_audio_tokens": 0, - "cache_creation_tokens": 0, - "cache_creation_token_details": CacheCreationTokenDetails( + result: Final = _cache_write_cost( + cache_creation_tokens=0, + cache_creation_token_details=CacheCreationTokenDetails( ephemeral_5m_input_tokens=100, ephemeral_1h_input_tokens=200, ), - "text_tokens": 0, - "audio_tokens": 0, - "image_tokens": 0, - "video_tokens": 0, - "character_count": 0, - "image_count": 0, - "video_length_seconds": 0.0, - "audio_length_seconds": 0.0, - "query_count": 0, - } - - model_info: ModelInfo = {} - - result = _calculate_input_cost( - prompt_tokens_details=prompt_tokens_details, - model_info=model_info, - prompt_base_cost=0.0, - cache_read_cost=0.0, - cache_creation_cost=cache_creation_cost, - cache_creation_cost_by_ttl={"1hr": cache_creation_cost_above_1hr, "12hr": cache_creation_cost_above_1hr}, + cache_creation_cost=3.75e-6, + cache_creation_cost_by_ttl={"1hr": 6e-6, "12hr": 6e-6}, ) - - # Expected: (100 * 3.75e-06) + (200 * 6e-06) = 0.000375 + 0.0012 = 0.001575 - expected = (100 * cache_creation_cost) + (200 * cache_creation_cost_above_1hr) - assert result > 0, "Cost should not be zero when ephemeral token details are present" - assert round(result, 6) == round(expected, 6) + assert result == pytest.approx(100 * 3.75e-6 + 200 * 6e-6) def test_service_tier_ultrafast_pricing(): @@ -2686,7 +2694,6 @@ def test_get_cost_per_unit_falls_back_from_service_tier_key_to_base(): ) def test_flat_input_price_keeps_other_token_thresholds(field: str) -> None: from typing import Final, cast - from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost info: Final = cast( ModelInfo, @@ -2698,21 +2705,20 @@ def test_flat_input_price_keeps_other_token_thresholds(field: str) -> None: f"{field}_above_300k_tokens": 0.005, }, ) - rates: Final = _get_token_base_cost( + rates: Final = _projected_rates( model_info=info, usage=Usage(prompt_tokens=300_001, completion_tokens=3, total_tokens=300_004), ) - assert rates.input_rate == 0.001 - assert rates.output_rate == (0.005 if field == "output_cost_per_token" else 0.002) - assert rates.cache_creation_rate == (0.005 if field == "cache_creation_input_token_cost" else 0.004) - assert rates.cache_read_rate == (0.005 if field == "cache_read_input_token_cost" else 0.003) + assert rates.input_cost_per_token == 0.001 + assert rates.output_cost_per_token == (0.005 if field == "output_cost_per_token" else 0.002) + assert rates.cache_creation_input_token_cost == (0.005 if field == "cache_creation_input_token_cost" else 0.004) + assert rates.cache_read_input_token_cost == (0.005 if field == "cache_read_input_token_cost" else 0.003) @pytest.mark.parametrize("prompt_tokens", (200_000, 200_001, 512_000, 512_001)) @pytest.mark.parametrize("higher_rate", (0.0, 0.009)) def test_mixed_token_thresholds_preserve_each_crossed_rate(prompt_tokens: int, higher_rate: float) -> None: from typing import Final, cast - from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost info: Final = cast( ModelInfo, @@ -2728,22 +2734,21 @@ def test_mixed_token_thresholds_preserve_each_crossed_rate(prompt_tokens: int, h "output_cost_per_token_above_512k_tokens": higher_rate, }, ) - rates: Final = _get_token_base_cost( + rates: Final = _projected_rates( model_info=info, usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=3, total_tokens=prompt_tokens + 3), ) - assert rates.input_rate == (0.005 if prompt_tokens > 200_000 else 0.001) - assert rates.output_rate == ( + assert rates.input_cost_per_token == (0.005 if prompt_tokens > 200_000 else 0.001) + assert rates.output_cost_per_token == ( higher_rate if prompt_tokens > 512_000 else 0.006 if prompt_tokens > 200_000 else 0.002 ) - assert rates.cache_creation_rate == (0.007 if prompt_tokens > 200_000 else 0.004) - assert rates.cache_read_rate == (higher_rate if prompt_tokens > 512_000 else 0.003) + assert rates.cache_creation_input_token_cost == (0.007 if prompt_tokens > 200_000 else 0.004) + assert rates.cache_read_input_token_cost == (higher_rate if prompt_tokens > 512_000 else 0.003) def test_threshold_keys_exclude_service_tier_variants(): from typing import cast - from litellm.litellm_core_utils.llm_cost_calc.utils import _get_token_base_cost from litellm.types.utils import ModelInfo, Usage # The service-tier-suffixed above-threshold key must be excluded from @@ -2761,7 +2766,7 @@ def test_threshold_keys_exclude_service_tier_variants(): }, ) usage = Usage(prompt_tokens=350_000, completion_tokens=1_000, total_tokens=351_000) - assert _get_token_base_cost(model_info=model_info, usage=usage).input_rate == 3e-6 + assert _projected_rates(model_info=model_info, usage=usage).input_cost_per_token == 3e-6 @pytest.mark.parametrize( @@ -3866,7 +3871,7 @@ def test_generic_cost_per_token_bills_cache_creation_at_the_input_rate_without_a ), ), ) -def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_path( +def test_billed_token_rates_resolves_missing_cache_write_rates_like_the_tiered_path( cache_rates: Mapping[str, float | Mapping[str, float | str]], current_time: datetime | None, expected_creation: float, @@ -3875,16 +3880,16 @@ def test_get_token_base_cost_resolves_missing_cache_write_rates_like_the_tiered_ model_info = {"input_cost_per_token": 2e-7, "output_cost_per_token": 1.25e-6, **cache_rates} usage = Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11) - rates = _get_token_base_cost(model_info, usage, current_time=current_time) + rates = _projected_rates(model_info, usage, current_time=current_time) - assert rates.cache_creation_rate == pytest.approx(expected_creation) - assert calculate_cache_writing_cost( + assert rates.cache_creation_input_token_cost == pytest.approx(expected_creation) + assert _cache_write_cost( cache_creation_tokens=3, cache_creation_token_details=CacheCreationTokenDetails( ephemeral_1h_input_tokens=1, ephemeral_12h_input_tokens=2 ), - cache_creation_cost=rates.cache_creation_rate, - cache_creation_cost_by_ttl=rates.cache_creation_rates_by_ttl, + cache_creation_cost=rates.cache_creation_input_token_cost, + cache_creation_cost_by_ttl=rates.cache_creation_input_token_cost_by_ttl, ) == pytest.approx(3 * expected_creation_1h) @@ -3977,14 +3982,33 @@ def test_route_image_generation_cost_bills_deployment_image_price_when_unlisted_ assert cost == pytest.approx(0.10) +def _batch_projected_rates(model_info: ModelInfo, usage: Usage, provider: str) -> Mapping[str, float | None]: + from litellm._cost import billing_context, evaluate, token_usage + from litellm.rust_bridge import _native + + result: Final = evaluate( + {"litellm_provider": provider, **model_info}, + token_usage(usage, model_info, batch=True), + billing_context(datetime(2026, 1, 1, tzinfo=timezone.utc), channel="batch"), + project=( + "input_cost_per_token", + "output_cost_per_token", + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr", + "cache_creation_input_token_cost_above_12hr", + ), + ) + assert not isinstance(result, _native.RowErrors), result + return {rate.requested_meter: rate.unit_price for rate in result.rates} + + def _batch_rates_model_info(**rates: object) -> ModelInfo: return cast(ModelInfo, dict(rates)) -def test_get_batch_cost_rates_parses_string_rates(): - from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - - rates = get_batch_cost_rates( +def test_batch_projected_rates_parses_string_rates(): + rates = _batch_projected_rates( _batch_rates_model_info( input_cost_per_token_batches="1e-06", output_cost_per_token_batches="4e-06", @@ -3997,34 +4021,37 @@ def test_get_batch_cost_rates_parses_string_rates(): "openai", ) - assert (rates.input, rates.output, rates.cache_read) == (2e-6, 6e-6, 2e-7) - - -def test_get_batch_cost_rates_falls_back_to_the_flat_rates_when_tier_rates_are_unparsable(): - from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - - rates = get_batch_cost_rates( - _batch_rates_model_info( - input_cost_per_token_batches=1e-6, - output_cost_per_token_batches=4e-6, - cache_read_input_token_cost_batches=1e-7, - input_cost_per_token_above_272k_tokens_batches="two", - output_cost_per_token_above_272k_tokens_batches="six", - cache_read_input_token_cost_above_272k_tokens_batches="none", - cache_creation_input_token_cost_batches=1.25e-7, - cache_creation_input_token_cost_above_272k_tokens_batches="nope", - ), - Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), - "openai", + assert (rates["input_cost_per_token"], rates["output_cost_per_token"], rates["cache_read_input_token_cost"]) == ( + 2e-6, + 6e-6, + 2e-7, ) - assert (rates.input, rates.output, rates.cache_read, rates.cache_creation) == (1e-6, 4e-6, 1e-7, 1.25e-7) + +def test_batch_pricing_rejects_unparsable_threshold_rates() -> None: + from litellm.cost_calculator import batch_cost_calculator + + info: Final = _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + output_cost_per_token_batches=4e-6, + cache_read_input_token_cost_batches=1e-7, + input_cost_per_token_above_272k_tokens_batches="two", + output_cost_per_token_above_272k_tokens_batches="six", + cache_read_input_token_cost_above_272k_tokens_batches="none", + cache_creation_input_token_cost_batches=1.25e-7, + cache_creation_input_token_cost_above_272k_tokens_batches="nope", + ) + with pytest.raises(litellm.ModelNotMappedError, match="above_272k"): + batch_cost_calculator( + Usage(prompt_tokens=300_000, completion_tokens=1), + "openai/batch-rate-validation", + "openai", + model_info=info, + ) -def test_get_batch_cost_rates_has_no_cached_rate_without_a_cached_batch_key(): - from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - - rates = get_batch_cost_rates( +def test_batch_projected_rates_cache_reads_fall_back_to_explicit_batch_input(): + rates = _batch_projected_rates( _batch_rates_model_info( input_cost_per_token_batches=1e-6, output_cost_per_token_batches=4e-6, @@ -4035,14 +4062,16 @@ def test_get_batch_cost_rates_has_no_cached_rate_without_a_cached_batch_key(): "openai", ) - assert (rates.input, rates.output, rates.cache_read) == (2e-6, 4e-6, None) + assert (rates["input_cost_per_token"], rates["output_cost_per_token"], rates["cache_read_input_token_cost"]) == ( + 2e-6, + 4e-6, + 2e-6, + ) @pytest.mark.parametrize(("prompt_tokens", "expected"), [(1_000, 1.25e-7), (272_000, 1.25e-7), (300_000, 2.5e-7)]) -def test_get_batch_cost_rates_reads_the_batch_cache_write_rate_for_the_crossed_tier(prompt_tokens, expected): - from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - - rates = get_batch_cost_rates( +def test_batch_projected_rates_reads_the_batch_cache_write_rate_for_the_crossed_tier(prompt_tokens, expected): + rates = _batch_projected_rates( _batch_rates_model_info( input_cost_per_token_batches=1e-7, input_cost_per_token_above_272k_tokens_batches=2e-7, @@ -4053,21 +4082,19 @@ def test_get_batch_cost_rates_reads_the_batch_cache_write_rate_for_the_crossed_t "openai", ) - assert rates.cache_creation == expected + assert rates["cache_creation_input_token_cost"] == expected @pytest.mark.parametrize( ("tier_key", "attribute"), [ - ("output_cost_per_token_above_272k_tokens_batches", "output"), - ("cache_read_input_token_cost_above_272k_tokens_batches", "cache_read"), - ("cache_creation_input_token_cost_above_272k_tokens_batches", "cache_creation"), + ("output_cost_per_token_above_272k_tokens_batches", "output_cost_per_token"), + ("cache_read_input_token_cost_above_272k_tokens_batches", "cache_read_input_token_cost"), + ("cache_creation_input_token_cost_above_272k_tokens_batches", "cache_creation_input_token_cost"), ], ) -def test_get_batch_cost_rates_crosses_a_tier_declared_without_an_input_tier_key(tier_key, attribute): - from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - - rates = get_batch_cost_rates( +def test_batch_projected_rates_crosses_a_tier_declared_without_an_input_tier_key(tier_key, attribute): + rates = _batch_projected_rates( _batch_rates_model_info( input_cost_per_token_batches=1e-6, output_cost_per_token_batches=4e-6, @@ -4079,18 +4106,16 @@ def test_get_batch_cost_rates_crosses_a_tier_declared_without_an_input_tier_key( "openai", ) - assert getattr(rates, attribute) == 9e-6 - assert rates.input == 1e-6 + assert rates[attribute] == 9e-6 + assert rates["input_cost_per_token"] == 1e-6 @pytest.mark.parametrize( ("prompt_tokens", "expected_input", "expected_output"), [(200_000, 1e-6, 4e-6), (250_000, 1e-6, 5e-6), (300_000, 2e-6, 5e-6)], ) -def test_get_batch_cost_rates_crosses_each_components_own_tier(prompt_tokens, expected_input, expected_output): - from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - - rates = get_batch_cost_rates( +def test_batch_projected_rates_crosses_each_components_own_tier(prompt_tokens, expected_input, expected_output): + rates = _batch_projected_rates( _batch_rates_model_info( input_cost_per_token_batches=1e-6, input_cost_per_token_above_272k_tokens_batches=2e-6, @@ -4101,13 +4126,11 @@ def test_get_batch_cost_rates_crosses_each_components_own_tier(prompt_tokens, ex "openai", ) - assert (rates.input, rates.output) == (expected_input, expected_output) + assert (rates["input_cost_per_token"], rates["output_cost_per_token"]) == (expected_input, expected_output) -def test_get_batch_cost_rates_has_no_cache_write_rate_without_a_cache_write_batch_key(): - from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - - rates = get_batch_cost_rates( +def test_batch_projected_rates_cache_writes_fall_back_to_explicit_batch_input(): + rates = _batch_projected_rates( _batch_rates_model_info( input_cost_per_token_batches=1e-7, input_cost_per_token_above_272k_tokens_batches=2e-7, @@ -4118,16 +4141,40 @@ def test_get_batch_cost_rates_has_no_cache_write_rate_without_a_cache_write_batc "openai", ) - assert rates.cache_creation is None + assert rates["cache_creation_input_token_cost"] == 2e-7 + + +@pytest.mark.parametrize( + ("meter", "details"), + ( + ("cache_read_input_token_cost", PromptTokensDetailsWrapper(cached_tokens=300_000)), + ("cache_creation_input_token_cost", PromptTokensDetailsWrapper(cache_creation_tokens=300_000)), + ), +) +def test_batch_cache_projection_agrees_with_billing_at_the_explicit_input_fallback( + meter: str, details: PromptTokensDetailsWrapper +) -> None: + from litellm.cost_calculator import batch_cost_calculator + + info: Final = _batch_rates_model_info( + input_cost_per_token_batches=1e-6, + input_cost_per_token_above_272k_tokens_batches=2e-6, + output_cost_per_token_batches=4e-6, + **{meter: 1e-7, f"{meter}_above_272k_tokens": 2e-7}, + ) + usage: Final = Usage(prompt_tokens=300_000, completion_tokens=1, prompt_tokens_details=details) + rates: Final = _batch_projected_rates(info, usage, "openai") + assert rates[meter] == 2e-6 + assert batch_cost_calculator(usage, "openai/batch-cache-fallback", "openai", model_info=info) == pytest.approx( + (0.6, 4e-6) + ) @pytest.mark.parametrize(("prompt_tokens", "expected"), [(1_000, 2.5e-7), (272_000, 2.5e-7), (300_000, 5e-7)]) -def test_get_batch_cost_rates_reads_the_12h_batch_cache_write_rate_for_the_crossed_tier( +def test_batch_projected_rates_reads_the_12h_batch_cache_write_rate_for_the_crossed_tier( prompt_tokens: int, expected: float ) -> None: - from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - - rates: Final = get_batch_cost_rates( + rates: Final = _batch_projected_rates( _batch_rates_model_info( input_cost_per_token_batches=1e-7, cache_creation_input_token_cost_batches=1.25e-7, @@ -4138,14 +4185,12 @@ def test_get_batch_cost_rates_reads_the_12h_batch_cache_write_rate_for_the_cross "openai", ) - assert rates.cache_creation == 1.25e-7 - assert rates.cache_creation_rates_by_ttl == {"12hr": expected} + assert rates["cache_creation_input_token_cost"] == 1.25e-7 + assert rates["cache_creation_input_token_cost_above_12hr"] == expected -def test_get_batch_cost_rates_has_no_12h_cache_write_rate_without_a_12h_batch_key() -> None: - from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - - rates: Final = get_batch_cost_rates( +def test_batch_projected_rates_has_no_12h_cache_write_rate_without_a_12h_batch_key() -> None: + rates: Final = _batch_projected_rates( _batch_rates_model_info( input_cost_per_token_batches=1e-7, cache_creation_input_token_cost_batches=1.25e-7, @@ -4156,7 +4201,7 @@ def test_get_batch_cost_rates_has_no_12h_cache_write_rate_without_a_12h_batch_ke "openai", ) - assert rates.cache_creation_rates_by_ttl == {} + assert rates["cache_creation_input_token_cost_above_12hr"] == 1.25e-7 @pytest.mark.parametrize("model_base", ["gpt-6-sol", "gpt-6-luna"]) @@ -4317,21 +4362,24 @@ def test_batch_cost_calculator_claude_haiku_5_5_prompt_length_tiers( (100_001, (2.5e-07, 1.25e-06, 2.5e-08, 3.125e-07)), ], ) -def test_get_batch_cost_rates_claude_haiku_5_5_prompt_length_tiers( +def test_batch_projected_rates_claude_haiku_5_5_prompt_length_tiers( _local_model_cost_map: None, prompt_tokens: int, expected: tuple[float, float, float, float], ) -> None: """Cache write and cache read batch rates are 50% of the standard rates; Anthropic's batch table omits them.""" - from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates - - rates: Final = get_batch_cost_rates( + rates: Final = _batch_projected_rates( litellm.get_model_info(model="claude-haiku-5-5", custom_llm_provider="anthropic"), Usage(prompt_tokens=prompt_tokens, completion_tokens=1, total_tokens=prompt_tokens + 1), "anthropic", ) - assert (rates.input, rates.output, rates.cache_read, rates.cache_creation) == expected + assert ( + rates["input_cost_per_token"], + rates["output_cost_per_token"], + rates["cache_read_input_token_cost"], + rates["cache_creation_input_token_cost"], + ) == expected @pytest.mark.parametrize(("speed", "multiplier"), (("fast", 6.0), ("standard", 1.0), (None, 1.0))) @@ -4425,3 +4473,137 @@ def test_token_breakdown_uses_supplied_pricing_without_catalog_lookup(rate: floa 10 * breakdown.rates.input_cost_per_token + breakdown.cache_read_cost + breakdown.cache_creation_cost ) assert output_cost == pytest.approx(3 * breakdown.rates.output_cost_per_token) + + +@pytest.mark.parametrize(("hour", "input_rate", "read_rate"), ((1, 1.0, 0.1), (12, 2.0, 0.25))) +@pytest.mark.parametrize("uplift", (1.0, 1.1)) +def test_all_cache_usage_reports_unused_input_rate_and_savings_at_pinned_time( + monkeypatch: pytest.MonkeyPatch, hour: int, input_rate: float, read_rate: float, uplift: float +) -> None: + model: Final = "openai/m5-all-cache-projection" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2.0, + "output_cost_per_token": 4.0, + "cache_read_input_token_cost": 0.25, + "regional_processing_uplift_multiplier_eu": uplift, + "off_peak_pricing": { + "hours_utc": "00:00-06:00", + "input_cost_per_token": 1.0, + "cache_read_input_token_cost": 0.1, + }, + }, + ) + usage: Final = Usage( + prompt_tokens=100, completion_tokens=0, prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100) + ) + when: Final = datetime(2026, 1, 1, hour, tzinfo=timezone.utc) + with pinned_billing_time(when): + charged: Final = litellm.cost_per_token( + model=model, + custom_llm_provider="openai", + prompt_tokens=100, + completion_tokens=0, + usage_object=usage, + data_residency="eu", + ) + breakdown: Final = get_token_type_cost_breakdown(model, "openai", usage, data_residency="eu") + savings: Final = calculate_prompt_caching_savings( + litellm.get_model_info(model, custom_llm_provider="openai"), + usage, + "openai", + data_residency="eu", + billed_at=when, + ) + assert charged == pytest.approx((100 * read_rate * uplift, 0.0)) + assert breakdown.rates is not None + assert breakdown.rates.input_cost_per_token == pytest.approx(input_rate * uplift) + assert breakdown.rates.output_cost_per_token == pytest.approx(4.0 * uplift) + assert breakdown.cache_read_cost == pytest.approx(charged[0]) + assert savings == pytest.approx(100 * (input_rate - read_rate) * uplift) + + +@pytest.mark.parametrize( + ("write_rate", "ttl_rate", "expected"), + ((None, None, 9.0), (0.0, 0.0, 19.0), (0.0, 2.0, -1.0), (0.5, 0.5, 19.0), (1.5, 2.0, -6.0)), +) +def test_cache_savings_preserve_base_and_ttl_write_premiums( + write_rate: float | None, ttl_rate: float | None, expected: float +) -> None: + info: Final = { + "input_cost_per_token": 1.0, + "cache_read_input_token_cost": 0.1, + "cache_creation_input_token_cost": write_rate, + "cache_creation_input_token_cost_above_1hr": ttl_rate, + } + usage: Final = Usage( + prompt_tokens=100, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=10, + cache_creation_tokens=20, + cache_creation_token_details=CacheCreationTokenDetails(ephemeral_1h_input_tokens=10), + ), + ) + assert calculate_prompt_caching_savings(info, usage, "custom") == pytest.approx(expected) + + +@pytest.mark.parametrize("explicit_rate", (None, 0.0, 0.25, 2.0)) +def test_explicit_cache_savings_use_the_billed_read_meter(explicit_rate: float | None) -> None: + info: Final = { + "input_cost_per_token": 1.0, + "cache_read_input_token_cost": 0.1, + "explicit_cache_read_input_token_cost": explicit_rate, + } + usage: Final = Usage( + prompt_tokens=100, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100, cache_type="ephemeral"), + ) + charged, _ = generic_cost_per_token("explicit-cache", usage, "custom", model_info=info) + savings: Final = calculate_prompt_caching_savings(info, usage, "custom") + assert savings == pytest.approx(max(100.0 - charged, 0.0)) + + +def test_cache_savings_bill_ttl_writes_even_without_the_aggregate_count() -> None: + info: Final = {"input_cost_per_token": 1.0, "cache_creation_input_token_cost_above_1hr": 1.5} + usage: Final = Usage( + prompt_tokens=0, + prompt_tokens_details=PromptTokensDetailsWrapper( + cache_creation_tokens=0, + cache_creation_token_details=CacheCreationTokenDetails(ephemeral_1h_input_tokens=20), + ), + ) + assert calculate_prompt_caching_savings(info, usage, "custom") == pytest.approx(-10.0) + + +@pytest.mark.parametrize("output_fields", ({}, {"output_cost_per_token": None}, {"output_cost_per_token": 0.0})) +def test_unused_missing_output_rate_is_not_reported_as_free( + monkeypatch: pytest.MonkeyPatch, output_fields: Mapping[str, float | None] +) -> None: + model: Final = "openai/m5-input-only-projection" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1.0, + **output_fields, + }, + ) + usage: Final = Usage(prompt_tokens=10, completion_tokens=0) + charged: Final = litellm.cost_per_token( + model=model, + custom_llm_provider="openai", + prompt_tokens=10, + completion_tokens=0, + usage_object=usage, + ) + rates: Final = get_billed_token_rates(model, "openai", usage) + assert charged == (10.0, 0.0) + assert rates is not None + assert rates.input_cost_per_token == 1.0 + assert rates.output_cost_per_token == output_fields.get("output_cost_per_token") diff --git a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py index 551caab7153..67304915b0b 100644 --- a/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py +++ b/tests/unit/llms/dashscope/test_dashscope_cost_calculator.py @@ -11,13 +11,14 @@ Tests the cost calculation for Dashscope models including: import math import os from datetime import datetime, timezone +from typing import Final import pytest # Add the project root to Python path import litellm +from litellm.exceptions import ModelNotMappedError from litellm.litellm_core_utils.llm_cost_calc.utils import ( - resolve_explicit_cache_read_cost_rate, calculate_prompt_caching_savings, generic_cost_per_token, get_token_type_cost_breakdown, @@ -43,32 +44,35 @@ class TestDashscopeCostCalculator: litellm.model_cost = litellm.get_model_cost_map(url="") @pytest.mark.parametrize( - "prompt_tokens_details", + ("prompt_tokens_details", "expected"), [ - {"cached_tokens": 0, "cache_type": "ephemeral"}, - {"cached_tokens": 600}, - {"cached_tokens": 600, "cache_type": "unknown-future-type"}, + ({"cached_tokens": 0, "cache_type": "ephemeral"}, 0.001), + ({"cached_tokens": 600}, 0.00052), + ({"cached_tokens": 600, "cache_type": "unknown-future-type"}, 0.00052), + ({"cached_tokens": 600, "cache_type": "ephemeral"}, 0.00046), ], ) def test_explicit_cache_rate_requires_an_ephemeral_cache_hit( self, prompt_tokens_details: dict[str, int | str], - ): - assert ( - resolve_explicit_cache_read_cost_rate( - model_info={ - "key": "qwen-cache-rate-guard-test", - "explicit_cache_read_input_token_cost": 1e-07, - }, - usage=Usage( - prompt_tokens=1000, - completion_tokens=0, - prompt_tokens_details=prompt_tokens_details, - ), - current_time=None, - ) - is None - ) + expected: float, + ) -> None: + assert generic_cost_per_token( + model="qwen-cache-rate-guard-test", + custom_llm_provider="dashscope", + model_info={ + "key": "qwen-cache-rate-guard-test", + "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 2e-07, + "explicit_cache_read_input_token_cost": 1e-07, + }, + usage=Usage( + prompt_tokens=1000, + completion_tokens=0, + prompt_tokens_details=prompt_tokens_details, + ), + current_time=None, + ) == pytest.approx((expected, 0)) @pytest.mark.parametrize( ("model_info", "current_time", "expected_cache_rate"), @@ -100,7 +104,7 @@ class TestDashscopeCostCalculator: ], }, None, - None, + 2e-07, ), ( { @@ -109,7 +113,7 @@ class TestDashscopeCostCalculator: "cache_read_input_token_cost": 2e-07, }, None, - None, + 2e-07, ), ( { @@ -179,7 +183,7 @@ class TestDashscopeCostCalculator: }, }, datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), - 8e-08, + None, ), ( { @@ -191,34 +195,49 @@ class TestDashscopeCostCalculator: }, }, datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc), - 1e-07, + None, ), ], ) - def test_provider_neutral_cache_rate_resolver_handles_qwen_rate_fallbacks( + def test_qwen_cache_mode_uses_native_fallbacks_and_rejects_invalid_rows( self, model_info: ModelInfo, current_time: datetime | None, expected_cache_rate: float | None, - ): - usage = Usage( + ) -> None: + usage: Final = Usage( prompt_tokens=1000, completion_tokens=0, prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"}, ) - cache_rate = resolve_explicit_cache_read_cost_rate( + if expected_cache_rate is None: + with pytest.raises(ModelNotMappedError): + generic_cost_per_token( + model="qwen-cache-rates", + custom_llm_provider="dashscope", + model_info=model_info, + usage=usage, + current_time=current_time, + ) + return + cost: Final = generic_cost_per_token( + model="qwen-cache-rates", + custom_llm_provider="dashscope", model_info=model_info, usage=usage, current_time=current_time, ) - assert cache_rate == expected_cache_rate + assert cost == pytest.approx((400e-06 + 600 * expected_cache_rate, 0)) - def test_provider_neutral_cache_rate_resolver_handles_service_tier_rate(self): - cache_rate = resolve_explicit_cache_read_cost_rate( + def test_ephemeral_cache_uses_the_service_tier_rate(self) -> None: + cost: Final = generic_cost_per_token( + model="qwen-service-tier-mode-rate-test", + custom_llm_provider="dashscope", model_info={ "key": "qwen-service-tier-mode-rate-test", + "input_cost_per_token": 1e-06, "explicit_cache_read_input_token_cost_priority": 3e-07, }, usage=Usage( @@ -230,7 +249,7 @@ class TestDashscopeCostCalculator: service_tier="priority", ) - assert cache_rate == 3e-07 + assert cost == pytest.approx((0.00058, 0)) def test_dashscope_flat_pricing_fallback(self): """ @@ -239,9 +258,7 @@ class TestDashscopeCostCalculator: """ usage = Usage(prompt_tokens=1000, completion_tokens=500) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-max", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-max", usage=usage) model_info = litellm.get_model_info("dashscope/qwen-max") expected_prompt_cost = 1000 * model_info["input_cost_per_token"] @@ -527,9 +544,7 @@ class TestDashscopeCostCalculator: """ # Tier 1 for qwen-flash is [0, 256,000] tokens usage = Usage(prompt_tokens=100000, completion_tokens=50000) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-flash", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage) model_info = litellm.get_model_info("dashscope/qwen-flash") tier_1_pricing = model_info["tiered_pricing"][0] @@ -547,9 +562,7 @@ class TestDashscopeCostCalculator: """ # Tiering for qwen-flash: Tier 1: [0, 256k], Tier 2: [256k, 1M] usage = Usage(prompt_tokens=300000, completion_tokens=300000) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-flash", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage) model_info = litellm.get_model_info("dashscope/qwen-flash") tier_1 = model_info["tiered_pricing"][0] @@ -561,9 +574,7 @@ class TestDashscopeCostCalculator: assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10) - graduated_prompt_cost = (256000 * tier_1["input_cost_per_token"]) + ( - 44000 * tier_2["input_cost_per_token"] - ) + graduated_prompt_cost = (256000 * tier_1["input_cost_per_token"]) + (44000 * tier_2["input_cost_per_token"]) assert prompt_cost > graduated_prompt_cost def test_dashscope_tiered_pricing_boundary_stays_in_lower_tier(self): @@ -572,18 +583,12 @@ class TestDashscopeCostCalculator: official `0 < Token <= 256K` phrasing. """ usage = Usage(prompt_tokens=256000, completion_tokens=1000) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-flash", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage) tier_1 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][0] - assert math.isclose( - prompt_cost, 256000 * tier_1["input_cost_per_token"], rel_tol=1e-10 - ) - assert math.isclose( - completion_cost, 1000 * tier_1["output_cost_per_token"], rel_tol=1e-10 - ) + assert math.isclose(prompt_cost, 256000 * tier_1["input_cost_per_token"], rel_tol=1e-10) + assert math.isclose(completion_cost, 1000 * tier_1["output_cost_per_token"], rel_tol=1e-10) def test_dashscope_tiered_pricing_output_uses_input_selected_tier(self): """ @@ -595,9 +600,7 @@ class TestDashscopeCostCalculator: tier_1 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][0] - assert math.isclose( - completion_cost, 400000 * tier_1["output_cost_per_token"], rel_tol=1e-10 - ) + assert math.isclose(completion_cost, 400000 * tier_1["output_cost_per_token"], rel_tol=1e-10) def test_dashscope_tiered_pricing_with_caching(self): """ @@ -626,17 +629,13 @@ class TestDashscopeCostCalculator: """ Requests above the highest declared range bill entirely at the last tier's rate. """ - usage = Usage( - prompt_tokens=1200000, completion_tokens=1000 - ) # Max defined range for qwen-flash is 1M + usage = Usage(prompt_tokens=1200000, completion_tokens=1000) # Max defined range for qwen-flash is 1M prompt_cost, _ = dashscope_cost_per_token(model="qwen-flash", usage=usage) tier_2 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][1] - assert math.isclose( - prompt_cost, 1200000 * tier_2["input_cost_per_token"], rel_tol=1e-10 - ) + assert math.isclose(prompt_cost, 1200000 * tier_2["input_cost_per_token"], rel_tol=1e-10) def _register_tiered_model(self, model_key: str, tiered_pricing: list[dict]) -> None: litellm.model_cost[model_key] = { @@ -671,9 +670,7 @@ class TestDashscopeCostCalculator: self._register_string_valued_tiered_model("dashscope/qwen-str-tier-test") usage = Usage(prompt_tokens=500, completion_tokens=200) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-str-tier-test", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-str-tier-test", usage=usage) assert math.isclose(prompt_cost, 500 * float("4e-07"), rel_tol=1e-10) assert math.isclose(completion_cost, 200 * float("1.6e-06"), rel_tol=1e-10) @@ -686,9 +683,7 @@ class TestDashscopeCostCalculator: self._register_string_valued_tiered_model("dashscope/qwen-str-tier-test") usage = Usage(prompt_tokens=2500, completion_tokens=3000) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-str-tier-test", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-str-tier-test", usage=usage) assert math.isclose(prompt_cost, 2500 * float("8e-07"), rel_tol=1e-10) assert math.isclose(completion_cost, 3000 * float("3.2e-06"), rel_tol=1e-10) @@ -721,18 +716,12 @@ class TestDashscopeCostCalculator: usage = Usage( prompt_tokens=300000, # 200k new + 60k cache creation + 40k cache read completion_tokens=1000, - prompt_tokens_details=PromptTokensDetailsWrapper( - cached_tokens=40000, cache_creation_tokens=60000 - ), + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=40000, cache_creation_tokens=60000), ) - prompt_cost, _ = dashscope_cost_per_token( - model="qwen-cache-write-test", usage=usage - ) + prompt_cost, _ = dashscope_cost_per_token(model="qwen-cache-write-test", usage=usage) - expected_prompt_cost = ( - (200000 * 6.5e-07) + (60000 * 8.125e-07) + (40000 * 6.5e-08) - ) + expected_prompt_cost = (200000 * 6.5e-07) + (60000 * 8.125e-07) + (40000 * 6.5e-08) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) @@ -769,13 +758,9 @@ class TestDashscopeCostCalculator: completion_tokens_details={"reasoning_tokens": 170}, ) - prompt_cost, _ = dashscope_cost_per_token( - model="qwen-nested-cache-write-test", usage=usage - ) + prompt_cost, _ = dashscope_cost_per_token(model="qwen-nested-cache-write-test", usage=usage) - assert math.isclose( - prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10 - ) + assert math.isclose(prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10) def test_dashscope_tiered_cache_creation_falls_back_to_tier_input_rate(self): """ @@ -799,9 +784,7 @@ class TestDashscopeCostCalculator: prompt_tokens_details=PromptTokensDetailsWrapper(cache_creation_tokens=4000), ) - prompt_cost, _ = dashscope_cost_per_token( - model="qwen-no-cache-write-test", usage=usage - ) + prompt_cost, _ = dashscope_cost_per_token(model="qwen-no-cache-write-test", usage=usage) assert math.isclose(prompt_cost, 10000 * 3.25e-07, rel_tol=1e-10) @@ -819,18 +802,12 @@ class TestDashscopeCostCalculator: usage = Usage( prompt_tokens=10000, completion_tokens=100, - prompt_tokens_details=PromptTokensDetailsWrapper( - cached_tokens=2000, cache_creation_tokens=3000 - ), + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=2000, cache_creation_tokens=3000), ) - prompt_cost, _ = dashscope_cost_per_token( - model="qwen-flat-cache-write-test", usage=usage - ) + prompt_cost, _ = dashscope_cost_per_token(model="qwen-flat-cache-write-test", usage=usage) - expected_prompt_cost = ( - (5000 * 3.25e-07) + (3000 * 4.063e-07) + (2000 * 3.25e-08) - ) + expected_prompt_cost = (5000 * 3.25e-07) + (3000 * 4.063e-07) + (2000 * 3.25e-08) assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10) @@ -847,9 +824,7 @@ class TestDashscopeCostCalculator: } usage = Usage(prompt_tokens=500, completion_tokens=200) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-input-only-tier-test", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-input-only-tier-test", usage=usage) assert math.isclose(prompt_cost, 500 * 4e-07, rel_tol=1e-10) assert math.isclose(completion_cost, 200 * 1.6e-06, rel_tol=1e-10) @@ -872,13 +847,9 @@ class TestDashscopeCostCalculator: completion_tokens=200, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), ) - _, completion_cost = dashscope_cost_per_token( - model="qwen-input-only-reasoning-test", usage=usage - ) + _, completion_cost = dashscope_cost_per_token(model="qwen-input-only-reasoning-test", usage=usage) - assert math.isclose( - completion_cost, (50 * 1.6e-06) + (150 * 4e-06), rel_tol=1e-10 - ) + assert math.isclose(completion_cost, (50 * 1.6e-06) + (150 * 4e-06), rel_tol=1e-10) def test_dashscope_tier_output_rate_wins_over_the_model_reasoning_rate(self): """ @@ -903,9 +874,7 @@ class TestDashscopeCostCalculator: completion_tokens=200, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), ) - _, completion_cost = dashscope_cost_per_token( - model="qwen-tier-output-reasoning-test", usage=usage - ) + _, completion_cost = dashscope_cost_per_token(model="qwen-tier-output-reasoning-test", usage=usage) assert math.isclose(completion_cost, 200 * 1.6e-06, rel_tol=1e-10) @@ -927,9 +896,7 @@ class TestDashscopeCostCalculator: completion_tokens=200, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), ) - _, completion_cost = dashscope_cost_per_token( - model="qwen-zero-reasoning-test", usage=usage - ) + _, completion_cost = dashscope_cost_per_token(model="qwen-zero-reasoning-test", usage=usage) assert math.isclose(completion_cost, 50 * 1.6e-06, rel_tol=1e-10) @@ -956,9 +923,7 @@ class TestDashscopeCostCalculator: completion_tokens=200, completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150), ) - _, completion_cost = dashscope_cost_per_token( - model="qwen-tier-zero-reasoning-test", usage=usage - ) + _, completion_cost = dashscope_cost_per_token(model="qwen-tier-zero-reasoning-test", usage=usage) assert math.isclose(completion_cost, 50 * 1.6e-06, rel_tol=1e-10) @@ -987,9 +952,7 @@ class TestDashscopeCostCalculator: } usage = Usage(prompt_tokens=0, completion_tokens=500) - prompt_cost, completion_cost = dashscope_cost_per_token( - model="qwen-zero-input-test", usage=usage - ) + prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-zero-input-test", usage=usage) assert prompt_cost == 0.0 assert math.isclose(completion_cost, 500 * 1.6e-06, rel_tol=1e-10) diff --git a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py b/tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py index c21943cfe75..44e59f454ab 100644 --- a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py +++ b/tests/unit/llms/fireworks_ai/test_fireworks_ai_cache_pricing.py @@ -1,65 +1,76 @@ +from collections.abc import Mapping from copy import deepcopy +from datetime import datetime, timezone +from typing import Final import pytest -from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO -from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate -from litellm.types.utils import ModelInfo +import litellm +from litellm.exceptions import ModelNotMappedError +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, get_billed_token_rates +from litellm.types.utils import Usage -def test_explicit_cache_read_rate_and_missing_input_rate_keep_the_entry_untouched() -> None: - explicit_info: ModelInfo = {"input_cost_per_token": 2e-6, "cache_read_input_token_cost": 1e-6} - no_input_rate_info: ModelInfo = {"output_cost_per_token": 3e-6} - - assert with_default_cache_read_rate(explicit_info) is explicit_info - assert with_default_cache_read_rate(no_input_rate_info) is no_input_rate_info +@pytest.mark.parametrize( + ("extra", "expected"), + ( + ({}, 0.0001), + ({"cache_read_input_token_cost": None}, 0.0001), + ({"cache_read_input_token_cost": 0.0}, 0.0), + ({"cache_read_input_token_cost": 0.3e-6}, 0.00003), + ({"off_peak_pricing": {"hours_utc": "14:00-00:00", "input_cost_per_token": 1e-6}}, 0.00005), + ({"off_peak_pricing": {"hours_utc": "14:00-00:00", "output_cost_per_token": 3e-6}}, 0.0001), + ({"off_peak_pricing": {"hours_utc": "14:00-00:00", "input_cost_per_token": "1e-6"}}, 0.00005), + ( + { + "off_peak_pricing": { + "hours_utc": "14:00-00:00", + "input_cost_per_token": 1e-6, + "cache_read_input_token_cost": None, + } + }, + 0.0001, + ), + ({"tiered_pricing": [{"range": [0, 200], "input_cost_per_token": 4e-6}]}, 0.0004), + ), +) +def test_fireworks_default_cache_rate_matches_projected_rates_without_mutating_the_row( + monkeypatch: pytest.MonkeyPatch, + extra: Mapping[str, object], + expected: float, +) -> None: + model: Final = "fireworks_ai/cache-default-test" + row: Final = {"input_cost_per_token": "2e-6", "litellm_provider": "fireworks_ai", "mode": "chat", **extra} + original: Final = deepcopy(row) + monkeypatch.setitem(litellm.model_cost, model, row) + model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="fireworks_ai") + usage: Final = Usage(prompt_tokens=100, completion_tokens=0, prompt_tokens_details={"cached_tokens": 100}) + when: Final = datetime(2026, 1, 1, 16, tzinfo=timezone.utc) + assert generic_cost_per_token( + model, + usage, + "fireworks_ai", + model_info=model_info, + current_time=when, + ) == pytest.approx((expected, 0)) + assert get_billed_token_rates( + model, + "fireworks_ai", + usage, + model_info=model_info, + current_time=when, + ).cache_read_input_token_cost == pytest.approx(expected / 100) + assert row == original -def test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry() -> None: - model_info: ModelInfo = { - "input_cost_per_token": 2e-6, - "off_peak_pricing": { - "hours_utc": "14:00-00:00", - "input_cost_per_token": 1e-6, - "output_cost_per_token": 3e-6, - }, - } - original: ModelInfo = deepcopy(model_info) - - derived = with_default_cache_read_rate(model_info) - - assert model_info == original - assert derived is not model_info - assert derived["cache_read_input_token_cost"] == pytest.approx(2e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO) - assert derived["off_peak_pricing"] == { - "hours_utc": "14:00-00:00", - "input_cost_per_token": 1e-6, - "output_cost_per_token": 3e-6, - "cache_read_input_token_cost": 1e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO, - } - - -def test_off_peak_window_without_its_own_input_rate_reuses_the_standard_derived_rate() -> None: - model_info: ModelInfo = { - "input_cost_per_token": 2e-6, - "off_peak_pricing": {"hours_utc": "14:00-00:00", "output_cost_per_token": 3e-6}, - } - - derived = with_default_cache_read_rate(model_info) - - assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == derived["cache_read_input_token_cost"] - - -def test_string_rates_from_config_are_coerced_before_the_discount_is_applied() -> None: - model_info: ModelInfo = { - "input_cost_per_token": "2e-6", - "off_peak_pricing": { - "hours_utc": "14:00-00:00", - "input_cost_per_token": "1e-6", - }, - } - - derived = with_default_cache_read_rate(model_info) - - assert derived["cache_read_input_token_cost"] == pytest.approx(1e-6) - assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == pytest.approx(5e-7) +def test_fireworks_missing_input_rate_remains_unknown_without_a_cache_price(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "fireworks_ai/cache-unknown-test" + row: Final = {"litellm_provider": "fireworks_ai", "mode": "chat", "output_cost_per_token": 3e-6} + monkeypatch.setitem(litellm.model_cost, model, row) + with pytest.raises(ModelNotMappedError): + generic_cost_per_token( + model, + Usage(prompt_tokens=100, completion_tokens=0, prompt_tokens_details={"cached_tokens": 100}), + "fireworks_ai", + ) + assert row == {"litellm_provider": "fireworks_ai", "mode": "chat", "output_cost_per_token": 3e-6} diff --git a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py b/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py index c3aec979aa6..56663741825 100644 --- a/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py +++ b/tests/unit/llms/fireworks_ai/test_fireworks_ai_cost_calculator.py @@ -323,10 +323,8 @@ def test_an_entry_without_an_input_rate_gets_no_cache_read_fallback(): } usage: Final = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200) - prompt_cost, completion_cost = cost_per_token(model="accounts/fireworks/models/no-input-rate-test", usage=usage) - - assert prompt_cost == 0 - assert completion_cost == 200 * 2e-06 + with pytest.raises(litellm.ModelNotMappedError, match="input_cost_per_token"): + cost_per_token(model="accounts/fireworks/models/no-input-rate-test", usage=usage) ROUTED_MODEL: Final = next( diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_priority_selection.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_priority_selection.py index 24309ed0460..c35d048e817 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_priority_selection.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_sampling_priority_selection.py @@ -7,8 +7,12 @@ per the MCP spec. """ from types import SimpleNamespace +from typing import Final from unittest.mock import patch +import pytest + +import litellm from litellm.proxy._experimental.mcp_server.sampling_handler import ( _has_priorities, _resolve_model_from_preferences, @@ -30,6 +34,14 @@ def _prefs(*, hints=None, cost=None, speed=None, intelligence=None): ) +def test_priority_reader_tolerates_missing_token_prices(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "openai/m5-sampling-reader" + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat"}) + info: Final = litellm.get_model_info(model) + assert (info["input_cost_per_token"], info["output_cost_per_token"]) == (None, None) + assert _select_model_by_priority([model], _prefs(cost=1.0)) == model + + # Model info stubs keyed by model name _MODEL_INFO = { "gpt-3.5-turbo": { diff --git a/tests/unit/proxy/auth/test_unmapped_model_budget_enforcement.py b/tests/unit/proxy/auth/test_unmapped_model_budget_enforcement.py index c2851f608f0..6f5d04fe2f6 100644 --- a/tests/unit/proxy/auth/test_unmapped_model_budget_enforcement.py +++ b/tests/unit/proxy/auth/test_unmapped_model_budget_enforcement.py @@ -8,17 +8,77 @@ See: https://github.com/BerriAI/litellm/issues/24770 """ import copy +from collections.abc import Mapping +from typing import Final import pytest import litellm -from litellm.proxy.auth.auth_checks import is_model_cost_zero +from litellm._catalog import is_priced +from litellm.proxy.auth.auth_checks import is_model_cost_zero, model_has_no_cost_mapping from litellm.router import Router - UNPRICED_ZERO_COST_MODEL = "ollama/unpriced-zero-cost-target" +@pytest.mark.parametrize( + ("prices", "present", "free"), + ( + ({}, False, False), + ({"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, True, True), + ({"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}, True, False), + ({"input_cost_per_token": 0.0}, True, False), + ({"output_cost_per_token": 2.0}, True, False), + ({"cache_read_input_token_cost": 0.0}, True, False), + ( + { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "ptu_count": 10, + "cost_per_ptu_per_hour": 1.0, + }, + True, + False, + ), + ( + { + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 3.0, "output_cost_per_token": 4.0}], + }, + True, + False, + ), + ), + ids=( + "metadata-only", + "explicit-free", + "paid", + "input-only-zero", + "output-only-positive", + "cache-only-zero", + "ptu", + "flat-plus-tier", + ), +) +def test_native_price_presence_does_not_replace_free_budget_policy( + monkeypatch: pytest.MonkeyPatch, prices: Mapping[str, object], present: bool, free: bool +) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + router: Final = Router( + model_list=[ + { + "model_name": "m5-budget-group", + "litellm_params": {"model": "openai/m5-budget-backend", "api_key": "fake-key"}, + "model_info": {"id": "m5-budget-deployment", **prices}, + } + ] + ) + assert is_priced("m5-budget-deployment") is present + assert model_has_no_cost_mapping("m5-budget-group", router) is not present + assert is_model_cost_zero("m5-budget-group", router) is free + + def _explicitly_free(name: str, model: str = "gpt-3.5-turbo", **model_info: float) -> dict: return { "model_name": name, diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 024fe5229be..a74e3dd3060 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -1,6 +1,7 @@ from __future__ import annotations import math +from collections.abc import Mapping from datetime import datetime, timedelta, timezone from typing import Final @@ -25,6 +26,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( ) from litellm.proxy.spend_tracking.budget_reservation import ( _get_team_member_budget_counter, + estimate_request_input_cost, estimate_request_max_cost, get_budget_window_start, release_unbound_budget_reservation, @@ -197,6 +199,68 @@ def _tiered_deployment(input_cost_per_token: float) -> Deployment: TIERED_BODY: Final = {"model": "tiered-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10} +@pytest.mark.parametrize( + ("prices", "expected_input", "expected_max"), + ( + ({}, None, None), + ({"input_cost_per_token": None, "output_cost_per_token": None}, None, None), + ({"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, 0.0, 0.0), + ({"input_cost_per_token": 2.0}, 20.0, None), + ), + ids=("missing", "null", "explicit-zero", "input-only"), +) +def test_reservation_reader_preserves_unknown_token_prices( + monkeypatch: pytest.MonkeyPatch, + prices: Mapping[str, float | None], + expected_input: float | None, + expected_max: float | None, +) -> None: + model: Final = "openai/m5-reservation-reader" + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **prices}) + request: Final = {"model": model, "messages": [{"role": "user", "content": "hello"}], "max_tokens": 5} + assert estimate_request_input_cost(request, "/chat/completions", None, {model: 10}) == expected_input + assert estimate_request_max_cost(request, "/chat/completions", None, {model: 10}) == expected_max + + +@pytest.mark.parametrize("tiered", (False, True)) +def test_reservation_keeps_the_reasoning_max_and_output_multiplier_without_discounts( + monkeypatch: pytest.MonkeyPatch, tiered: bool +) -> None: + model: Final = "openai/m5-conservative-reservation" + prices: Final = { + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "output_cost_per_reasoning_token": 5.0, + } + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "chat", + **({"tiered_pricing": [{"range": [0, 100], **prices}]} if tiered else prices), + "input_cost_per_token_flex": 0.1, + "output_cost_per_token_flex": 0.2, + "off_peak_pricing": { + "hours_utc": "00:00-00:00", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "output_cost_per_reasoning_token": 0.05, + }, + }, + ) + request: Final = { + "model": model, + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 5, + "service_tier": "flex", + "n": 2, + "best_of": 3, + } + assert estimate_request_input_cost(request, "/chat/completions", None, {model: 10}) == 10.0 + assert estimate_request_max_cost(request, "/chat/completions", None, {model: 10}) == 85.0 + + def test_repeated_estimates_reuse_cached_model_cost_info() -> None: router: Final = Router(model_list=[_tiered_deployment(1e-06).model_dump()]) first: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router) diff --git a/tests/unit/proxy/spend_tracking/test_savings.py b/tests/unit/proxy/spend_tracking/test_savings.py index f9566fd1ba0..b0f685ffebe 100644 --- a/tests/unit/proxy/spend_tracking/test_savings.py +++ b/tests/unit/proxy/spend_tracking/test_savings.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping +from datetime import datetime, timezone from typing import Final, Literal import pytest @@ -137,6 +139,81 @@ def test_compression_savings_priced_at_input_rate(): assert result.prompt_caching == 0.0 +@pytest.mark.parametrize( + ("prices", "saved_tokens", "expected"), + ( + ({}, 10, 0.0), + ({"input_cost_per_token": None}, 10, 0.0), + ({"input_cost_per_token": 0.0}, 10, 0.0), + ({"input_cost_per_token": 2.0}, -10, 0.0), + ({"input_cost_per_token": 2.0}, 10, 20.0), + ), +) +def test_compression_keeps_the_flat_reporting_estimate_instead_of_token_selection( + monkeypatch: pytest.MonkeyPatch, prices: Mapping[str, float | None], saved_tokens: int, expected: float +) -> None: + model: Final = "openai/compression-policy-test" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "chat", + **prices, + "output_cost_per_token": 1.0, + "tiered_pricing": [{"range": [0, 2000], "input_cost_per_token": 0.1, "output_cost_per_token": 0.1}], + "input_cost_per_token_priority": 0.01, + "off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 0.001}, + }, + ) + result: Final = compute_savings_spend( + model, + "openai", + saved_tokens, + False, + usage_object={"prompt_tokens": 1000, "completion_tokens": 0}, + cost_breakdown={"service_tier": "priority"}, + billed_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + ) + assert result.compression == expected + assert result.prompt_caching == 0.0 + + +def test_compression_reporting_does_not_leak_deployment_prices(monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "openai/compression-policy-test" + row: Final = { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1.0, + "output_cost_per_token": 1.0, + } + monkeypatch.setitem(litellm.model_cost, model, dict(row)) + router: Final = Router( + model_list=[ + { + "model_name": "compression-a", + "litellm_params": {"model": model, "input_cost_per_token": 2.0}, + "model_info": {"id": "compression-a"}, + }, + { + "model_name": "compression-b", + "litellm_params": {"model": model, "input_cost_per_token": 3.0}, + "model_info": {"id": "compression-b"}, + }, + ] + ) + + def estimate(model_id: str) -> float: + return compute_savings_spend( + model, "openai", 10, False, model_id=model_id, llm_router=lambda: router + ).compression + + assert estimate("compression-a") == 20.0 + assert estimate("compression-b") == 30.0 + assert estimate("compression-a") == 20.0 + assert litellm.model_cost[model] == row + + def test_prompt_caching_savings_priced_at_input_minus_cache_read(): input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5") # A model that supports prompt caching must charge less to read from cache; @@ -635,8 +712,6 @@ def test_equal_modeled_usage_is_zero_under_equivalent_model_names() -> None: assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage, usage) == 0.0 - - def test_unresolvable_baseline_remains_unknown(): usage = _usage(fresh=2000, cached=0, written=0, out=500) assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) is None diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 64b415b3175..a53e8791c6f 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -13,8 +13,11 @@ from openai.types.completion_usage import CompletionUsage from pydantic import BaseModel import litellm +from litellm.exceptions import ModelNotMappedError from litellm import TranscriptionResponse, model_cost from litellm.constants import REALTIME_METERED_USAGE_KEY + + from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details from litellm.integrations.custom_logger import CustomLogger @@ -43,6 +46,30 @@ import threading from unittest.mock import MagicMock, patch +@pytest.mark.parametrize("provider", (None, "", 123, {})) +def test_registered_bedrock_pricing_does_not_replace_a_declared_provider( + monkeypatch: pytest.MonkeyPatch, provider: object +) -> None: + model: Final = "bedrock-invalid-provider-context" + litellm.register_model( + {model: {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}}, + custom_llm_provider="bedrock", + persist_across_reloads=False, + ) + monkeypatch.setitem(litellm.model_cost[model], "litellm_provider", provider) + if provider == "": + assert litellm.cost_per_token(model, 10, 20, custom_llm_provider="bedrock") == (10.0, 40.0) + assert litellm.get_model_info(model=model)["litellm_provider"] == "" + else: + with pytest.raises(ModelNotMappedError, match="litellm_provider"): + litellm.cost_per_token(model, 10, 20, custom_llm_provider="bedrock") + assert litellm.model_cost[model] == { + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "litellm_provider": provider, + } + + class ResponseCostCapture(CustomLogger): def __init__(self) -> None: self.success_cost: float | None = None @@ -493,7 +520,7 @@ def test_completion_cost_skips_huggingface_config_request_with_custom_pricing( assert cost == pytest.approx(usage.prompt_tokens * 0.001 + usage.completion_tokens * 0.002) -def test_completion_cost_returns_zero_for_capability_only_generalization( +def test_completion_cost_rejects_unknown_pricing_for_capability_only_generalization( _local_model_cost_map: None, _claude_family_generalization: None, monkeypatch: pytest.MonkeyPatch, @@ -506,13 +533,8 @@ def test_completion_cost_returns_zero_for_capability_only_generalization( usage=Usage(prompt_tokens=17, completion_tokens=9, total_tokens=26), ) - cost: Final = completion_cost( - completion_response=response, - model=model, - custom_llm_provider="bedrock", - ) - - assert cost == 0.0 + with pytest.raises(ModelNotMappedError, match="input_cost_per_token"): + completion_cost(completion_response=response, model=model, custom_llm_provider="bedrock") @pytest.mark.parametrize("custom_llm_provider", ["openai", "hosted_vllm"]) @@ -1842,7 +1864,15 @@ def test_cost_calculator_with_cache_creation(): } ), ) - model = "claude-sonnet-4@20250514" + model: Final = "claude-cache-creation-contract" + litellm.register_model({f"vertex_ai/{model}": { + "litellm_provider": "vertex_ai", + "mode": "chat", + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000003, + "cache_read_input_token_cost": 0.0000001, + "cache_creation_input_token_cost": 0.000004, + }}) assert litellm_model_response.usage.prompt_tokens_details.cached_tokens == 28491 @@ -1852,7 +1882,7 @@ def test_cost_calculator_with_cache_creation(): custom_llm_provider="vertex_ai", ) - print(result) + assert result == pytest.approx(13 * 0.000003 + 28491 * 0.0000001 + 15 * 0.000004) def test_bedrock_cost_calculator_comparison_with_without_cache(): @@ -7350,6 +7380,45 @@ def test_vertex_ai_medlm_completion_cost(): assert predictive_cost > 0 +@pytest.mark.parametrize( + ("direction", "threshold_field", "prompt_tokens", "completion_tokens", "expected"), + [ + ("output", None, 200001, 10, (20.0, 30.0)), + ("output", "output_cost_per_token_above_200k_tokens", 200001, 10, (20.0, 50.0)), + ("output", "output_cost_per_token_above_128k_tokens", 200001, 10, (20.0, 30.0)), + ("output", "output_cost_per_token_above_128k_tokens", 10, 128001, (20.0, 640005.0)), + ("input", None, 10, 200001, (30.0, 20.0)), + ("input", "input_cost_per_token_above_200k_tokens", 200001, 10, (1000005.0, 20.0)), + ("input", "input_cost_per_token_above_128k_tokens", 200001, 10, (1000005.0, 20.0)), + ], +) +def test_vertex_mixed_character_pricing_only_prices_the_used_token_direction( + direction: Literal["input", "output"], + threshold_field: str | None, + prompt_tokens: int, + completion_tokens: int, + expected: tuple[float, float], +) -> None: + model: Final = "vertex_ai/mixed-character-contract" + character_direction: Final = "input" if direction == "output" else "output" + litellm.register_model({model: { + "litellm_provider": "vertex_ai", + "mode": "chat", + f"{character_direction}_cost_per_character": 2.0, + f"{direction}_cost_per_token": 3.0, + **({threshold_field: 5.0} if threshold_field is not None else {}), + }}) + + assert cost_per_token( + model=model, + custom_llm_provider="vertex_ai", + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + prompt_characters=10 if character_direction == "input" else None, + completion_characters=10 if character_direction == "output" else None, + ) == expected + + def test_vertex_ai_embedding_completion_cost(caplog): """ Relevant issue - https://github.com/BerriAI/litellm/issues/4630 diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index f4b54731512..a03f3e1ad13 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -1,26 +1,16 @@ -""" -Test that register_model() in completion() and embedding() passes all -custom pricing fields from kwargs and model_info, not just the base -input/output costs. - -Previously, only input_cost_per_token, output_cost_per_token, and -litellm_provider were forwarded. Fields like cache_read_input_token_cost, -mode, and supports_prompt_caching were dropped, causing incorrect cost -calculations for DB-sourced models with prompt caching pricing. -""" - import copy import itertools import os from collections.abc import Callable, Mapping +from datetime import datetime, timezone +from queue import Queue from typing import Final, NoReturn import pytest - import litellm +from litellm._internal_context import pinned_billing_time from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map -from litellm.main import _build_custom_pricing_entry from litellm.utils import _invalidate_model_cost_lowercase_map @@ -28,6 +18,350 @@ def _snapshot_model_cost_entries(keys): return {key: copy.deepcopy(litellm.model_cost.get(key)) for key in keys} +@pytest.mark.parametrize("stream", (False, True)) +def test_sdk_custom_prices_do_not_leak_between_tenants_or_into_deployments( + monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + + model: Final = "openai/m5-tenant-backend" + deployment_id: Final = "m5-tenant-deployment" + catalog_row: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0} + owned_row: Final = {"litellm_provider": "openai", "input_cost_per_token": 7.0, "output_cost_per_token": 8.0} + monkeypatch.setitem(litellm.model_cost, model, catalog_row.copy()) + monkeypatch.setitem(litellm.model_cost, deployment_id, owned_row.copy()) + events: Final[Queue[tuple[object, object]]] = Queue() + + def on_success(kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime) -> None: + if kwargs.get("stream") and kwargs.get("complete_streaming_response") is None: + return + events.put((kwargs.get("response_cost"), kwargs.get("complete_streaming_response") or response)) + + monkeypatch.setattr(litellm, "success_callback", [on_success]) + for input_rate, output_rate, per_call in ( + (5.0, 6.0, {}), + ( + 1.0, + 4.0, + { + "input_cost_per_token": 1.0, + "output_cost_per_token": 4.0, + "metadata": {"model_info": {"id": deployment_id}}, + }, + ), + (5.0, 6.0, {}), + ): + result: Final = litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="Hello back", + stream=stream, + **per_call, + ) + if isinstance(result, CustomStreamWrapper): + assert tuple(result) + cost, response = events.get(timeout=10) + assert isinstance(response, litellm.ModelResponse) + assert isinstance(response.usage, litellm.Usage) + expected: Final = response.usage.prompt_tokens * input_rate + response.usage.completion_tokens * output_rate + assert cost == pytest.approx(expected) + if not stream: + assert response._hidden_params["response_cost"] == pytest.approx(expected) + assert litellm.model_cost[model] == catalog_row + assert litellm.model_cost[deployment_id] == owned_row + assert litellm.cost_per_token(deployment_id, 10, 20, custom_llm_provider="openai") == pytest.approx((70.0, 160.0)) + + +@pytest.mark.parametrize( + ("prices", "usage", "service_tier", "expected"), + ( + ( + {"output_cost_per_reasoning_token": 3.0}, + litellm.Usage(prompt_tokens=10, completion_tokens=10, completion_tokens_details={"reasoning_tokens": 5}), + "default", + 35.0, + ), + ( + {"input_cost_per_audio_token": 3.0, "output_cost_per_audio_token": 4.0}, + litellm.Usage( + prompt_tokens=10, + completion_tokens=10, + prompt_tokens_details={"text_tokens": 6, "audio_tokens": 4}, + completion_tokens_details={"text_tokens": 6, "audio_tokens": 4}, + ), + "default", + 46.0, + ), + ( + {"input_cost_per_token_priority": 3.0, "output_cost_per_token_priority": 4.0}, + litellm.Usage(prompt_tokens=10, completion_tokens=10), + "priority", + 70.0, + ), + ), + ids=("reasoning", "audio", "priority"), +) +def test_sdk_request_prices_preserve_non_base_token_fields_without_catalog_writes( + monkeypatch: pytest.MonkeyPatch, + prices: Mapping[str, float], + usage: litellm.Usage, + service_tier: str, + expected: float, +) -> None: + model: Final = "openai/m5-request-token-fields" + row: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0} + monkeypatch.setitem(litellm.model_cost, model, row.copy()) + response: Final = litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + input_cost_per_token=1.0, + output_cost_per_token=2.0, + mock_response=litellm.ModelResponse(model=model, usage=usage, service_tier=service_tier), + **prices, + ) + assert response._hidden_params["response_cost"] == pytest.approx(expected) + assert litellm.model_cost[model] == row + assert litellm.cost_per_token(model, 10, 10, custom_llm_provider="openai") == pytest.approx((50.0, 60.0)) + + +@pytest.mark.parametrize("stream", (False, True)) +def test_router_request_prices_override_only_the_current_call(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None: + from litellm.integrations.custom_logger import CustomLogger + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + + model: Final = "openai/m5-routed-request" + deployment_id: Final = "m5-routed-request-deployment" + catalog: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0} + monkeypatch.setitem(litellm.model_cost, model, catalog.copy()) + events: Final = Queue[tuple[object, object]]() + + class Costs(CustomLogger): + def log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + events.put((kwargs.get("response_cost"), kwargs.get("complete_streaming_response", response_obj))) + + monkeypatch.setattr(litellm, "callbacks", [Costs()]) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "m5-request-group", + "litellm_params": {"model": model, "api_key": "fake-key"}, + "model_info": { + "id": deployment_id, + "input_cost_per_token": 7.0, + "output_cost_per_token": 8.0, + }, + } + ], + num_retries=0, + ) + owned: Final = copy.deepcopy(litellm.model_cost[deployment_id]) + try: + for input_rate, output_rate, per_call in ( + (7.0, 8.0, {}), + (1.0, 2.0, {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}), + (0.0, 0.0, {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}), + (7.0, 8.0, {}), + ): + result: Final = router.completion( + model="m5-request-group", + messages=[{"role": "user", "content": "hello"}], + mock_response="Hello back", + stream=stream, + **per_call, + ) + if isinstance(result, CustomStreamWrapper): + assert tuple(result) + cost, response = events.get(timeout=10) + assert isinstance(response, litellm.ModelResponse) + assert isinstance(response.usage, litellm.Usage) + assert cost == pytest.approx( + response.usage.prompt_tokens * input_rate + response.usage.completion_tokens * output_rate + ) + assert litellm.model_cost[deployment_id] == owned + assert litellm.model_cost[model] == catalog + finally: + router.discard() + + +def test_request_prices_reach_callback_cache_and_reasoning_breakdown_without_registration( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.integrations.custom_logger import CustomLogger + + model: Final = "openai/m5-request-breakdown" + row: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0} + monkeypatch.setitem(litellm.model_cost, model, row.copy()) + events: Final = Queue[object]() + + class Costs(CustomLogger): + def log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + events.put(kwargs.get("standard_logging_object")) + + monkeypatch.setattr(litellm, "callbacks", [Costs()]) + response: Final = litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + input_cost_per_token=1.0, + output_cost_per_token=2.0, + cache_read_input_token_cost=0.1, + cache_creation_input_token_cost=1.25, + output_cost_per_reasoning_token=3.0, + model_info={"mode": "chat", "supports_prompt_caching": True, "max_tokens": 8192}, + mock_response=litellm.ModelResponse( + model=model, + usage=litellm.Usage( + prompt_tokens=100, + completion_tokens=10, + prompt_tokens_details={"cached_tokens": 60, "cache_creation_tokens": 20}, + completion_tokens_details={"reasoning_tokens": 5}, + ), + ), + ) + payload: Final = events.get(timeout=10) + assert isinstance(payload, dict) + assert payload["response_cost"] == pytest.approx(76.0) + assert payload["cost_breakdown"] == pytest.approx( + { + "input_cost": 51.0, + "output_cost": 25.0, + "total_cost": 76.0, + "tool_usage_cost": 0.0, + "service_tier": None, + "data_residency": None, + "vertex_location": None, + "cache_read_cost": 6.0, + "cache_creation_cost": 25.0, + "reasoning_cost": 15.0, + "original_cost": 76.0, + "discount_percent": 0.0, + "discount_amount": 0.0, + "margin_percent": 0.0, + "margin_fixed_amount": 0.0, + "margin_total_amount": 0.0, + } + ) + assert response._hidden_params["response_cost"] == pytest.approx(76.0) + assert litellm.model_cost[model] == row + + +@pytest.mark.parametrize(("cache_price", "expected"), ((None, 0.5), (0.0, 0.0), (0.25, 0.25))) +def test_request_all_cache_usage_retains_unused_rates_and_zero_overrides( + monkeypatch: pytest.MonkeyPatch, cache_price: float | None, expected: float +) -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + + model: Final = "openai/m5-request-all-cache" + row: Final = { + "litellm_provider": "openai", + "input_cost_per_token": 5.0, + "output_cost_per_token": 6.0, + "cache_read_input_token_cost": 0.5, + } + monkeypatch.setitem(litellm.model_cost, model, row.copy()) + when: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + logging_obj: Final = Logging( + model=model, + messages=[{"role": "user", "content": "hello"}], + stream=False, + call_type="completion", + start_time=when, + litellm_call_id="m5-request-all-cache", + function_id="f", + kwargs={ + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "cache_read_input_token_cost": cache_price, + }, + ) + with pinned_billing_time(when): + total: Final = litellm.completion_cost( + model=model, + custom_llm_provider="openai", + custom_pricing=True, + litellm_logging_obj=logging_obj, + completion_response=litellm.ModelResponse( + model=model, + usage=litellm.Usage( + prompt_tokens=100, completion_tokens=0, prompt_tokens_details={"cached_tokens": 100} + ), + ), + ) + assert total == pytest.approx(100 * expected) + assert logging_obj.billed_token_rates is not None + assert logging_obj.billed_token_rates.input_cost_per_token == 1.0 + assert logging_obj.billed_token_rates.output_cost_per_token == 2.0 + assert logging_obj.billed_token_rates.cache_read_input_token_cost == expected + assert litellm.model_cost[model] == row + + +@pytest.mark.parametrize(("speed", "geo", "factor"), (("standard", "global", 1.0), ("fast", "us", 6.6))) +def test_ephemeral_request_prices_keep_provider_selected_adjustments( + monkeypatch: pytest.MonkeyPatch, speed: str, geo: str, factor: float +) -> None: + model: Final = "anthropic/m5-request-adjustments" + row: Final = { + "litellm_provider": "anthropic", + "mode": "chat", + "input_cost_per_token": 5.0, + "output_cost_per_token": 6.0, + "provider_specific_entry": {"fast": 6.0, "us": 1.1}, + } + monkeypatch.setitem(litellm.model_cost, model, copy.deepcopy(row)) + response: Final = litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + input_cost_per_token=1.0, + output_cost_per_token=2.0, + mock_response=litellm.ModelResponse( + model=model, + usage=litellm.Usage(prompt_tokens=10, completion_tokens=10, speed=speed, inference_geo=geo), + ), + ) + assert response._hidden_params["response_cost"] == pytest.approx(30.0 * factor) + assert litellm.model_cost[model] == row + + +@pytest.mark.parametrize("field", ("input_cost_per_second", "output_cost_per_second", "cost_per_second")) +@pytest.mark.parametrize("rate", (0.0, 0.02)) +def test_request_duration_prices_keep_python_time_billing_without_registration(field: str, rate: float) -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + + model: Final = "openai/m5-request-duration" + logging_obj: Final = Logging( + model=model, + messages=[{"role": "user", "content": "hello"}], + stream=False, + call_type="completion", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="m5-request-duration", + function_id="f", + kwargs={field: rate}, + ) + total: Final = litellm.completion_cost( + model=model, + custom_llm_provider="openai", + custom_pricing=True, + litellm_logging_obj=logging_obj, + total_time=1500.0, + completion_response=litellm.ModelResponse( + model=model, usage=litellm.Usage(prompt_tokens=10, completion_tokens=10) + ), + ) + assert total == pytest.approx(rate * 1.5) + assert logging_obj.cost_breakdown is not None + assert logging_obj.cost_breakdown["input_cost"] == pytest.approx(rate * 1.5) + assert logging_obj.cost_breakdown["output_cost"] == 0.0 + assert model not in litellm.model_cost + + def _restore_model_cost_entries(original_entries): for key, value in original_entries.items(): if value is None: @@ -37,131 +371,6 @@ def _restore_model_cost_entries(original_entries): _invalidate_model_cost_lowercase_map() -def test_build_custom_pricing_entry_includes_all_kwargs_fields(): - """All CustomPricingLiteLLMParams fields present in kwargs should be - included in the resulting entry dict.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - "cache_read_input_token_cost": 0.00025, - "cache_creation_input_token_cost": 0.005, - "output_cost_per_reasoning_token": 0.01, - "input_cost_per_audio_token": 0.003, - "unrelated_kwarg": "should_be_ignored", - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_token"] == 0.001 - assert entry["output_cost_per_token"] == 0.002 - assert entry["cache_read_input_token_cost"] == 0.00025 - assert entry["cache_creation_input_token_cost"] == 0.005 - assert entry["output_cost_per_reasoning_token"] == 0.01 - assert entry["input_cost_per_audio_token"] == 0.003 - assert "unrelated_kwarg" not in entry - - -def test_build_custom_pricing_entry_merges_model_info_metadata(): - """Fields from model_info (mode, supports_prompt_caching, max_tokens) - should be merged into the entry when present.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - model_info = { - "id": "deployment-123", - "mode": "chat", - "supports_prompt_caching": True, - "max_tokens": 128000, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=model_info, - ) - - assert entry["mode"] == "chat" - assert entry["supports_prompt_caching"] is True - assert entry["max_tokens"] == 128000 - - -def test_build_custom_pricing_entry_setdefault_does_not_override_existing(): - """model_info uses setdefault, so it should not override a key that is - already present in the entry dict. Currently CustomPricingLiteLLMParams - and the model_info keys (mode, supports_prompt_caching, max_tokens) do - not overlap, but if they ever do, setdefault ensures the kwargs-sourced - value wins.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - model_info = { - "mode": "chat", - "supports_prompt_caching": True, - "max_tokens": 128000, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=model_info, - ) - - assert entry["mode"] == "chat" - assert entry["supports_prompt_caching"] is True - assert entry["max_tokens"] == 128000 - - # Verify setdefault behavior: if a model_info key already exists in - # the entry (e.g. from a future CustomPricingLiteLLMParams addition), - # setdefault must not overwrite it. - entry["mode"] = "embedding" # simulate pre-existing value - # Re-apply setdefault the same way _build_custom_pricing_entry does - entry.setdefault("mode", model_info["mode"]) - assert entry["mode"] == "embedding" # must NOT revert to "chat" - - -def test_build_custom_pricing_entry_skips_none_values(): - """Fields with None values in kwargs should not be included.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": None, # explicitly None - "cache_read_input_token_cost": None, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["input_cost_per_token"] == 0.001 - assert "output_cost_per_token" not in entry - assert "cache_read_input_token_cost" not in entry - - -def test_build_custom_pricing_entry_handles_no_model_info(): - """Should work correctly when model_info is None.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=None, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_token"] == 0.001 - assert entry["output_cost_per_token"] == 0.002 - assert "mode" not in entry - - def test_register_model_receives_cache_pricing_fields(): """End-to-end: when register_model is called with a full pricing entry, the cache pricing fields should be present in litellm.model_cost.""" @@ -192,23 +401,6 @@ def test_register_model_receives_cache_pricing_fields(): litellm.model_cost.pop(model_key, None) -def test_build_custom_pricing_entry_time_based(): - """Time-based pricing fields should be included correctly.""" - kwargs = { - "input_cost_per_second": 0.01, - "output_cost_per_second": 0.02, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_second"] == 0.01 - assert entry["output_cost_per_second"] == 0.02 - - def test_register_model_strips_none_litellm_provider(): """``get_model_info`` returns ``litellm_provider: None`` for deployments registered without a provider (e.g. ``Router.add_deployment`` flows). @@ -598,6 +790,16 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): assert builtin_input_cost > 0 try: + litellm.register_model( + { + deployment_id: { + "litellm_provider": "openai", + "mode": "embedding", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + } + ) litellm.embedding( model=shared_key, input=["hello"], @@ -640,6 +842,16 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): assert builtin_input_cost != override_input_cost try: + litellm.register_model( + { + deployment_id: { + "litellm_provider": "openai", + "mode": "embedding", + "input_cost_per_token": override_input_cost, + "output_cost_per_token": override_input_cost * 2, + } + } + ) response = litellm.embedding( model=shared_key, input=["hello"], @@ -674,6 +886,16 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): assert builtin_input_cost > 0 try: + litellm.register_model( + { + deployment_id: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + } + ) litellm.completion( model=shared_key, messages=[{"role": "user", "content": "hello"}], @@ -693,11 +915,7 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): _restore_model_cost_entries(snapshot) -def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): - """Direct SDK calls (no router deployment id in metadata) keep the legacy - behavior: custom pricing is registered under ``{provider}/{model}`` and the - request is costed with it. - """ +def test_embedding_direct_sdk_custom_pricing_does_not_register_shared_key(): model_key = "openai/lit3991-direct-sdk-embed-model" override_input_cost = 3e-05 try: @@ -710,13 +928,18 @@ def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): mock_response=[0.1, 0.2], ) - assert litellm.model_cost[model_key]["input_cost_per_token"] == override_input_cost + assert model_key not in litellm.model_cost + assert response._hidden_params["response_cost"] == pytest.approx(10 * override_input_cost) cost = litellm.completion_cost( completion_response=response, model=model_key, custom_llm_provider="openai", call_type="embedding", custom_pricing=True, + custom_cost_per_token={ + "input_cost_per_token": override_input_cost, + "output_cost_per_token": override_input_cost * 2, + }, ) assert cost == pytest.approx(10 * override_input_cost) finally: @@ -1417,10 +1640,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing(): object stays untouched, and the caller's incoming nested dict is never inserted by reference into the merged result. """ - from litellm.utils import update_dictionary - from typing import cast + from litellm.utils import update_dictionary + existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"}) existing = {"off_peak_pricing": existing_nested} incoming_nested = cast( @@ -1619,12 +1842,12 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing(): del router -def test_completion_registers_cost_per_second_pricing(): +def test_completion_uses_ephemeral_cost_per_second_pricing(): model_key: Final = "openai/test-cost-per-second-registration" original_entries: Final = _snapshot_model_cost_entries([model_key]) try: - litellm.completion( + response: Final = litellm.completion( model=model_key, messages=[{"role": "user", "content": "hello"}], api_key="fake-key", @@ -1632,7 +1855,14 @@ def test_completion_registers_cost_per_second_pricing(): mock_response="hello back", ) - assert litellm.model_cost[model_key]["cost_per_second"] == 0.02 + assert model_key not in litellm.model_cost + assert response._hidden_params["response_cost"] is not None + assert litellm.completion_cost( + litellm.ModelResponse(model=model_key), + model=model_key, + custom_cost_per_second=0.02, + total_time=2000, + ) == pytest.approx(0.04) finally: _restore_model_cost_entries(original_entries) @@ -1686,9 +1916,7 @@ def test_register_model_scopes_builtin_match_to_the_given_provider(): _restore_model_cost_entries(model_cost_entries) -def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider(): - from litellm.main import _register_custom_pricing_for_request - +def test_per_request_custom_pricing_does_not_write_a_colliding_deployment_id(): deployment_id: Final = "baseten/zai-org/glm-5.2" builtin_key: Final = "baseten/zai-org/GLM-5.2" shared_key: Final = "openai/zai-org/GLM-5.2" @@ -1696,20 +1924,20 @@ def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_prov builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key]) openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models) try: - _register_custom_pricing_for_request( - model="zai-org/GLM-5.2", - custom_llm_provider="openai", - kwargs={ - "input_cost_per_token": 0.00000096, - "output_cost_per_token": 0.00000302, - "metadata": {"model_info": {"id": deployment_id}}, - }, - model_info={"mode": "chat"}, + response: Final = litellm.completion( + model=shared_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + input_cost_per_token=0.00000096, + output_cost_per_token=0.00000302, + metadata={"model_info": {"id": deployment_id}}, + mock_response=litellm.ModelResponse( + model=shared_key, usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30) + ), ) - entry: Final = litellm.model_cost[deployment_id] - assert entry["input_cost_per_token"] == 0.00000096 - assert entry["output_cost_per_token"] == 0.00000302 + assert response._hidden_params["response_cost"] == pytest.approx(10 * 0.00000096 + 20 * 0.00000302) + assert litellm.model_cost.get(deployment_id) == model_cost_entries[deployment_id] assert litellm.model_cost[builtin_key] == builtin_row_before finally: _restore_model_cost_entries(model_cost_entries) diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index f93bbb11bbe..1b23802b507 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -3161,12 +3161,10 @@ def test_inherit_builtin_tiered_output_rate_fills_the_backend_flat_rate(): def test_inherit_builtin_tiered_output_rate_never_stores_a_synthesized_zero(): """ - Regression: get_model_info reports output_cost_per_token 0 for a backend that - only publishes tiered rates (e.g. dashscope/qwen-flash), and storing that zero - would mark the deployment as explicitly priced free. + Missing flat prices must not mark a tier-priced backend as explicitly free. """ backend_info = litellm.get_model_info(model="qwen-flash", custom_llm_provider="dashscope") - assert backend_info["output_cost_per_token"] == 0 + assert backend_info["output_cost_per_token"] is None model_info = {"tiered_pricing": [{"range": [0, 3000], "input_cost_per_token": 3.25e-07}]} Router._inherit_builtin_tiered_output_rate( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 1eaabdc0adc..22bb87f2702 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2445,7 +2445,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None: @respx.mock @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") -def test_completion_registers_test_owned_model_prices() -> None: +def test_completion_uses_request_prices_without_registering_them() -> None: model: Final = "migration-owned-pricing-model" messages: Final = [{"role": "user", "content": "price control"}] litellm.register_model( @@ -2488,8 +2488,7 @@ def test_completion_registers_test_owned_model_prices() -> None: ) assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages} - assert litellm.model_cost[f"openai/{model}"]["input_cost_per_token"] == 0.001 - assert litellm.model_cost[f"openai/{model}"]["output_cost_per_token"] == 0.002 + assert f"openai/{model}" not in litellm.model_cost assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001 assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002 assert response._hidden_params["response_cost"] == pytest.approx(0.018)