diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql new file mode 100644 index 00000000000..50c9c269ead --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20261031000000_accounting_compilation_state/migration.sql @@ -0,0 +1,8 @@ +SET LOCAL lock_timeout = '10s'; + +ALTER TABLE "LiteLLM_AccountingEvent" +ADD COLUMN IF NOT EXISTS "compilation_state" TEXT NOT NULL DEFAULT 'pending', +ADD COLUMN IF NOT EXISTS "compilation_error" TEXT, +ADD COLUMN IF NOT EXISTS "compilation_retry_at" TIMESTAMP(3); +CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_kind_compilation_state_recorded_at_idx" +ON "LiteLLM_AccountingEvent"("kind", "compilation_state", "recorded_at"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 179b741fe7c..c2f34e3b17e 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -2111,8 +2111,12 @@ model LiteLLM_AccountingEvent { occurred_at DateTime recorded_at DateTime @default(now()) payload Json + compilation_state String @default("pending") + compilation_error String? + compilation_retry_at DateTime? @@index([recorded_at]) + @@index([kind, compilation_state, recorded_at]) @@index([kind, source_event_id]) } diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e66be233be4..9f567a6c707 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -26,6 +26,12 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR +from litellm.litellm_core_utils.llm_cost_calc.pricing_context import ( + pricing_discounts, + pricing_margins, + retained_model_info, + retained_models, +) from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -379,7 +385,9 @@ def _per_second_pricing_cost( audio_seconds: float = 0.0, ) -> tuple[float, float] | None: try: - model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = retained_model_info(model, custom_llm_provider) or cached_get_model_info_helper( + model=model, custom_llm_provider=custom_llm_provider + ) except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model return None if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): @@ -538,7 +546,7 @@ def cost_per_token( if response_cost is not None: return response_cost[0], response_cost[1] - model_cost_ref: Final = litellm.model_cost + model_cost_ref: Final = retained_models() or litellm.model_cost # Only callers that explicitly pass `custom_llm_provider` get the # dedup/prefix-join treatment. When provider is omitted, preserve legacy # behavior: `model_with_provider` stays equal to the raw `model` string @@ -600,9 +608,11 @@ def cost_per_token( model = model_without_prefix if is_decisions_call_type(call_type): - decisions_model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + decisions_model_info: Final = retained_model_info(model, custom_llm_provider) or cached_get_model_info_helper( + model=model, custom_llm_provider=custom_llm_provider + ) decisions_overrides: Final = decisions_pricing_fields( - litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded + (retained_models() or litellm.model_cost).get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded ) if decisions_overrides: return generic_cost_per_token( @@ -623,7 +633,9 @@ def cost_per_token( ) if lyria_generation_cost is not None: return 0.0, lyria_generation_cost - speech_model_info: Final = cached_get_model_info_helper( + speech_model_info: Final = retained_model_info( + model_without_prefix, custom_llm_provider + ) or cached_get_model_info_helper( model=model_without_prefix, custom_llm_provider=custom_llm_provider, preserve_unset_costs=True, @@ -808,7 +820,9 @@ def cost_per_token( service_tier=service_tier, ) else: - model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + model_info: Final = retained_model_info(model, custom_llm_provider) or cached_get_model_info_helper( + model=model, custom_llm_provider=custom_llm_provider + ) if _has_token_or_tiered_pricing(model_info): return generic_cost_per_token( model=model, @@ -995,8 +1009,8 @@ def select_model_name_for_cost_calc( ) if custom_pricing is True: - if router_model_id is not None and router_model_id in litellm.model_cost: - entry: Final = litellm.model_cost[router_model_id] + if router_model_id is not None and router_model_id in (retained_models() or litellm.model_cost): + entry: Final = (retained_models() or litellm.model_cost)[router_model_id] if _cost_map_entry_prices_anything(entry): return_model = router_model_id else: @@ -1075,14 +1089,16 @@ def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already resolves downstream (custom-priced router ids) or no stripped candidate is registered (#38069).""" segments: Final = model.split("/") - if "/".join(segments[1:]) in litellm.model_cost: + if "/".join(segments[1:]) in (retained_models() or litellm.model_cost): return model head_len: Final = 2 if region_name is not None and len(segments) > 2 and segments[1] == region_name else 1 head: Final = "/".join(segments[:head_len]) tail: Final = segments[head_len:] strippable: Final = next((index for index, segment in enumerate(tail) if segment in LlmProvidersSet), len(tail)) candidates: Final = (f"{head}/{'/'.join(tail[start:])}" for start in range(min(strippable, len(tail) - 1) + 1)) - return next((candidate for candidate in candidates if candidate in litellm.model_cost), model) + return next( + (candidate for candidate in candidates if candidate in (retained_models() or litellm.model_cost)), model + ) @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) @@ -1348,9 +1364,7 @@ def _apply_cost_discount( Tuple of (final_cost, discount_percent, discount_amount) """ original_cost: Final = base_cost - resolved_discount: Final = resolve_cost_discount( - litellm.cost_discount_config, custom_llm_provider, model, region_name - ) + resolved_discount: Final = resolve_cost_discount(pricing_discounts(), custom_llm_provider, model, region_name) discount_percent: Final = 0.0 if resolved_discount is None else resolved_discount discount_amount: Final = original_cost * discount_percent @@ -1389,12 +1403,12 @@ def _apply_cost_margin( # Get margin config - check provider-specific first, then global margin_config = None - if custom_llm_provider and custom_llm_provider in litellm.cost_margin_config: - margin_config = litellm.cost_margin_config[custom_llm_provider] + if custom_llm_provider and custom_llm_provider in pricing_margins(): + margin_config = pricing_margins()[custom_llm_provider] if verbose_logger.isEnabledFor(logging.DEBUG): verbose_logger.debug("Found provider-specific margin config for %s: %s", custom_llm_provider, margin_config) - elif "global" in litellm.cost_margin_config: - margin_config = litellm.cost_margin_config["global"] + elif "global" in pricing_margins(): + margin_config = pricing_margins()["global"] if verbose_logger.isEnabledFor(logging.DEBUG): verbose_logger.debug("Using global margin config: %s", margin_config) else: @@ -1402,7 +1416,7 @@ def _apply_cost_margin( verbose_logger.debug( "No margin config found. Provider: %s, Available configs: %s", custom_llm_provider, - list(litellm.cost_margin_config.keys()), + list(pricing_margins().keys()), ) if margin_config is not None: @@ -1995,7 +2009,7 @@ def completion_cost( video_resolution=video_resolution, ) elif call_type in _SPEECH_CALL_TYPES: - prompt_characters = _speech_billed_characters( + prompt_characters = speech_billed_characters( completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None, prompt, ) @@ -2159,12 +2173,14 @@ def completion_cost( # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai" - ) and not has_together_registry_pricing(model, litellm.model_cost): + ) and not has_together_registry_pricing(model, (retained_models() or litellm.model_cost)): model = get_model_params_and_category(model, call_type=CallTypes(call_type)) # replicate llms are calculate based on time for request running # see https://replicate.com/pricing - elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost: + elif (model in litellm.replicate_models or "replicate" in model) and model not in ( + retained_models() or litellm.model_cost + ): # for unmapped replicate model, default to replicate's time tracking logic return get_replicate_completion_pricing(completion_response, response_time_ms) @@ -2261,7 +2277,7 @@ def completion_cost( _final_cost += sum(additional_costs.values()) original_cost = _final_cost - if litellm.cost_discount_config: + if pricing_discounts(): ( _final_cost, discount_percent, @@ -2277,7 +2293,7 @@ def completion_cost( discount_amount = 0.0 # Apply margin from module-level config if configured - if litellm.cost_margin_config: + if pricing_margins(): ( _final_cost, margin_percent, @@ -2352,15 +2368,15 @@ def completion_cost( def get_response_cost_from_hidden_params( - hidden_params: dict | BaseModel, + hidden_params: Mapping[str, object] | BaseModel, ) -> float | None: if isinstance(hidden_params, BaseModel): _hidden_params_dict = hidden_params.model_dump() else: _hidden_params_dict = hidden_params - additional_headers: Final = _hidden_params_dict.get("additional_headers", {}) - if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers: + additional_headers: Final = as_str_mapping(_hidden_params_dict.get("additional_headers")) + if additional_headers is not None and "llm_provider-x-litellm-response-cost" in additional_headers: return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"]) return None @@ -2511,7 +2527,7 @@ def response_cost_calculator( raise e -def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int: +def speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int: reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt) @@ -2629,7 +2645,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, . def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: - raw_entry: Final = litellm.model_cost.get(key) + raw_entry: Final = (retained_models() or litellm.model_cost).get(key) return raw_entry if isinstance(raw_entry, Mapping) else None @@ -2908,6 +2924,46 @@ def transcription_cost(model: str, custom_llm_provider: str | None, duration: fl return openai_cost_per_second(model=model, custom_llm_provider=custom_llm_provider, duration=duration) +def image_cost_model_candidates( + model: str, custom_llm_provider: str | None, quality: str | None, size: str | None +) -> tuple[str | None, ...]: + # Standardize size format to use "-x-" + size_str: str = size or "1024-x-1024" + size_str = size_str.replace("x", "-x-") if "x" in size_str and "-x-" not in size_str else size_str + + # Build model names for cost lookup + base_model_name = f"{size_str}/{model}" + model_name_without_custom_llm_provider: str | None = None + if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"): + model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "") + base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}" + model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name + provider_first_model_name_with_quality: Final = ( + f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}" + if quality and custom_llm_provider + else None + ) + + # gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family + model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}" + + verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name) + + model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}" + model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider + + return ( + model_name_with_quality, + provider_first_model_name_with_quality, + base_model_name, + model_name_with_v2_quality, + model_with_quality_without_provider, + model_without_provider, + model, + model_name_without_custom_llm_provider, + ) + + def default_image_cost_calculator( model: str, custom_llm_provider: str | None = None, @@ -2934,51 +2990,23 @@ def default_image_cost_calculator( Raises: Exception: If model pricing not found in cost map """ - # Standardize size format to use "-x-" - size_str: str = size or "1024-x-1024" - size_str = size_str.replace("x", "-x-") if "x" in size_str and "-x-" not in size_str else size_str - - # Parse dimensions - height, width = map(int, size_str.split("-x-")) - - # Build model names for cost lookup - base_model_name = f"{size_str}/{model}" - model_name_without_custom_llm_provider: str | None = None - if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"): - model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "") - base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}" - model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name - provider_first_model_name_with_quality: Final = ( - f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}" - if quality and custom_llm_provider - else None - ) - - # gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family - model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}" - - verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name) - - model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}" - model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider - - models_to_check: Final = ( - model_name_with_quality, - provider_first_model_name_with_quality, - base_model_name, - model_name_with_v2_quality, - model_with_quality_without_provider, - model_without_provider, - model, - model_name_without_custom_llm_provider, - ) + size_str: Final = (size or "1024x1024").replace("-x-", "x") + height, width = map(int, size_str.split("x")) + models_to_check: Final = image_cost_model_candidates(model, custom_llm_provider, quality, size) matched_model: Final = next( - (_model for _model in models_to_check if _model is not None and _model in litellm.model_cost), None + ( + _model + for _model in models_to_check + if _model is not None and _model in (retained_models() or litellm.model_cost) + ), + None, ) if matched_model is None and model_info is None: raise Exception(f"Model not found in cost map. Tried checking {models_to_check}") - shared_cost_info: Final = litellm.model_cost[matched_model] if matched_model is not None else None + shared_cost_info: Final = ( + (retained_models() or litellm.model_cost)[matched_model] if matched_model is not None else None + ) price_tables: Final = tuple(table for table in (model_info, shared_cost_info) if table is not None) image_count: Final = n if n is not None else 1 unit_counts: Final = ( @@ -3016,7 +3044,7 @@ def default_video_cost_calculator( custom_llm_provider (Optional[str]): Custom LLM provider model_info (Optional[ModelInfo]): Deployment-level model info containing custom video pricing. When provided, used before falling back to - the global litellm.model_cost lookup. + the effective model cost lookup. video_resolution (Optional[str]): From usage (e.g. ``720p``, ``1080p``) for tiered per-second pricing. Returns: @@ -3026,9 +3054,9 @@ def default_video_cost_calculator( Exception: If model pricing not found in cost map """ # Use custom model_info pricing if provided (deployment-specific pricing) - cost_info: dict | None = None + cost_info: ModelInfo | None = None if model_info is not None: - cost_info = dict(model_info) + cost_info = model_info else: # Build model names for cost lookup base_model_name = model @@ -3049,15 +3077,15 @@ def default_video_cost_calculator( model_name_without_custom_llm_provider, ] for _model in models_to_check: - if _model is not None and _model in litellm.model_cost: - cost_info = litellm.model_cost[_model] + if _model is not None and _model in (retained_models() or litellm.model_cost): + cost_info = (retained_models() or litellm.model_cost)[_model] break # If still not found, try with custom_llm_provider prefix if cost_info is None and custom_llm_provider: prefixed_model: Final = f"{custom_llm_provider}/{model}" - if prefixed_model in litellm.model_cost: - cost_info = litellm.model_cost[prefixed_model] + if prefixed_model in (retained_models() or litellm.model_cost): + cost_info = (retained_models() or litellm.model_cost)[prefixed_model] if cost_info is None: raise Exception(f"Model not found in cost map for model={model}") @@ -3663,9 +3691,9 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) """ resolved: Final = _cost_map_model_info(model_name, custom_llm_provider) entries: Final = ( - litellm.model_cost.get(resolved.get("key")) if resolved is not None else None, - litellm.model_cost.get(model_name), - litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"), + (retained_models() or litellm.model_cost).get(resolved.get("key")) if resolved is not None else None, + (retained_models() or litellm.model_cost).get(model_name), + (retained_models() or litellm.model_cost).get(f"{custom_llm_provider}/{model_name}"), ) return any(entry is not None and _cost_map_entry_prices_anything(entry) for entry in entries) @@ -3910,11 +3938,11 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None: - """First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry + """First of ``keys`` this entry prices, read off the raw model cost entry because ``get_model_info`` synthesizes zero token rates for entries that omit them.""" if info is None: return None - declared: Final = litellm.model_cost.get(info.get("key")) + declared: Final = (retained_models() or litellm.model_cost).get(info.get("key")) if declared is None: return None return next( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 639003c9a6a..41d95903785 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -14,6 +14,7 @@ import sys import time import traceback from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence +from contextlib import nullcontext from dataclasses import dataclass from datetime import datetime as dt_object from enum import Enum @@ -25,7 +26,13 @@ from httpx import Response, Timeout from pydantic import BaseModel, JsonValue, TypeAdapter import litellm -from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase +from litellm._internal_context import ( + current_billing_time, + in_mirrored_call, + outside_mirrored_call, + pinned_billing_time, + post_response_phase, +) from litellm._logging import ( is_debugging_on, redact_string, @@ -54,9 +61,11 @@ from litellm.constants import ( from litellm.cost_calculator import ( RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, + get_response_cost_from_hidden_params, get_usage_object, pricing_entry_for_cost_calc, select_model_name_for_cost_calc, + speech_billed_characters, ) from litellm.exceptions import ( BudgetExceededError, @@ -78,6 +87,7 @@ from litellm.litellm_core_utils.classifier_logging import ( is_classifier_call, ) from litellm.litellm_core_utils.core_helpers import ( + billed_usage_cost, finite_cost, get_provider_response_headers_from_hidden_params, is_expected_client_error, @@ -112,6 +122,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( object_mapping, with_guardrail_entry, ) +from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -666,6 +677,10 @@ class UsageCaptureObservation: return cls(serialized, provenance, response_cost) +_REQUEST_MODEL_INFO: Final = TypeAdapter(ModelInfo) +_REQUEST_PRICING_MODELS: Final = TypeAdapter(dict[str, ModelInfo]) + + class Logging(LiteLLMLoggingBaseClass): usage_capture_callback: ClassVar[ Callable[ @@ -703,6 +718,8 @@ class Logging(LiteLLMLoggingBaseClass): logfireLogger, \ prometheusLogger, \ slack_app + cost_billing_time: datetime.datetime | None = None + request_pricing: RetainedPricing | None = None custom_pricing: bool = False custom_cost_per_token: CostPerToken | None = None custom_cost_per_second: float | None = None @@ -2107,6 +2124,70 @@ class Logging(LiteLLMLoggingBaseClass): if margin_total_amount is not None: self.cost_breakdown["margin_total_amount"] = margin_total_amount + def _request_pricing_model_info( + self, model: str, provider: str, overrides: Mapping[str, object], deployment_id: str | None + ) -> ModelInfo: + for candidate in (deployment_id, f"{provider}/{model}"): + if candidate is None: + continue + try: + return litellm.get_model_info(model=candidate, custom_llm_provider=provider) + except Exception: # noqa: BLE001 # unmapped SDK models can supply their complete pricing + verbose_logger.debug("No base pricing for request override candidate %s", candidate) + return _REQUEST_MODEL_INFO.validate_python( + { + "key": deployment_id or f"{provider}/{model}", + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": None, + "output_cost_per_token": None, + "litellm_provider": provider, + "mode": overrides.get( + "mode", "embedding" if "embedding" in str(self.model_call_details.get("call_type", "")) else "chat" + ), + "supported_openai_params": None, + } + ) + + def set_request_pricing( + self, + model: str, + provider: str, + overrides: Mapping[str, object] | None, + deployment_id: str | None = None, + ) -> None: + from litellm.utils import model_info_with_pricing_overrides + + if overrides is None: + self.request_pricing = None + return + base: Final = self._request_pricing_model_info(model, provider, overrides, deployment_id) + normalized: Final = _REQUEST_MODEL_INFO.validate_python( + { + **base, + "mode": base.get("mode") + or ("embedding" if "embedding" in str(self.model_call_details.get("call_type", "")) else "chat"), + } + ) + effective: Final = model_info_with_pricing_overrides( + normalized, + { + name: value + for name, value in overrides.items() + if deployment_id is None + or name not in {"input_cost_per_token", "output_cost_per_token"} + or normalized.get(name) != value + }, + ) + keys: Final = (model, f"{provider}/{model}", deployment_id or model, effective["key"]) + self.request_pricing = RetainedPricing( + models_json=_REQUEST_PRICING_MODELS.dump_json({name: effective for name in keys}).decode(), + discounts_json=json.dumps(litellm.cost_discount_config), + margins_json=json.dumps(litellm.cost_margin_config), + allow_live_models=True, + ) + def response_cost_calculator( self, result: object, @@ -2114,6 +2195,26 @@ class Logging(LiteLLMLoggingBaseClass): litellm_model_name: str | None = None, router_model_id: str | None = None, ignore_stamped_cost: bool = False, + custom_pricing_override: bool | None = None, + ) -> float | None: + context: Final = ( + retained_pricing(self.request_pricing) + if self.request_pricing is not None and not self.spilled_over_ptu_for(result) + else nullcontext() + ) + with context, pinned_billing_time(self.billing_time_for_cost()): + return self._calculate_response_cost( + result, cache_hit, litellm_model_name, router_model_id, ignore_stamped_cost, custom_pricing_override + ) + + def _calculate_response_cost( + self, + result: object, + cache_hit: bool | None = None, + litellm_model_name: str | None = None, + router_model_id: str | None = None, + ignore_stamped_cost: bool = False, + custom_pricing_override: bool | None = None, ) -> float | None: """ Calculate response cost using result + logging object variables. @@ -2176,7 +2277,9 @@ class Logging(LiteLLMLoggingBaseClass): router_model_id = self.get_router_model_id() ## RESPONSE COST ## - custom_pricing: Final = self.custom_pricing_for(priced_result) + custom_pricing: Final = ( + self.custom_pricing_for(priced_result) if custom_pricing_override is None else custom_pricing_override + ) prompt: Final = self._prompt_for_cost_calculation() @@ -2404,7 +2507,9 @@ class Logging(LiteLLMLoggingBaseClass): def custom_pricing_for(self, result: object) -> bool: if self.spilled_over_ptu_for(result): return False - return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None)) + return self.request_pricing is not None or use_custom_pricing_for_model( + litellm_params=getattr(self, "litellm_params", None) + ) def _prompt_for_cost_calculation(self) -> str: """ @@ -2965,6 +3070,8 @@ class Logging(LiteLLMLoggingBaseClass): metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY) return { "id": self.litellm_call_id, + "model": self.model, + "results": result, "usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")} if metered is not None else None, @@ -2974,15 +3081,31 @@ class Logging(LiteLLMLoggingBaseClass): ) return normalized if assembled is None else assembled - def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]: - from litellm.utils import count_characters + def billing_time_for_cost(self) -> datetime.datetime: + if self.cost_billing_time is None: + self.cost_billing_time = current_billing_time() + return self.cost_billing_time + def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]: return TypeAdapter(dict[str, JsonValue]).validate_python( { + "response_ms": getattr(result, "_response_ms", None), + "provider_reported_cost": self._provider_reported_cost + if self._provider_reported_cost is not None + else get_response_cost_from_hidden_params(dict(get_hidden_params(result) or {})), + "provider_usage_cost": billed_usage_cost(result, self.custom_llm_provider), + "current_billing_time": self.billing_time_for_cost().isoformat(), "custom_pricing": self.custom_pricing_for(result), "custom_cost_per_token": self.custom_cost_per_token, "custom_cost_per_second": self.custom_cost_per_second, - "prompt_characters": count_characters(self._prompt_for_cost_calculation()) + "tool_response_cost": self.model_call_details.get("response_cost") + if self.model_call_details.get("call_type") + in (CallTypes.call_mcp_tool.value, CallTypes.send_message.value, CallTypes.asend_message.value) + else None, + "prompt_characters": speech_billed_characters( + result if isinstance(result, HttpxBinaryResponseContent) else None, + self._prompt_for_cost_calculation(), + ) if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value) else None, } @@ -7838,14 +7961,20 @@ def get_standard_logging_object_payload( timing_metrics: Final = getattr(logging_obj, "response_timing_metrics", None) or {} clean_hidden_params["litellm_overhead_time_ms"] = timing_metrics.get("litellm_overhead_time_ms") - model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information( - base_model=base_model, - custom_pricing=custom_pricing, - custom_llm_provider=kwargs.get("custom_llm_provider"), - init_response_obj=init_response_obj, - api_base=litellm_params.get("api_base"), - router_model_id=logging_obj.get_router_model_id(), + pricing_context: Final = ( + retained_pricing(logging_obj.request_pricing) + if logging_obj.request_pricing is not None and not spilled_over + else nullcontext() ) + with pricing_context: + model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information( + base_model=base_model, + custom_pricing=custom_pricing, + custom_llm_provider=kwargs.get("custom_llm_provider"), + init_response_obj=init_response_obj, + api_base=litellm_params.get("api_base"), + router_model_id=logging_obj.get_router_model_id(), + ) error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload( metadata=metadata, diff --git a/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py b/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py new file mode 100644 index 00000000000..9cc9255cfeb --- /dev/null +++ b/litellm/litellm_core_utils/llm_cost_calc/pricing_context.py @@ -0,0 +1,84 @@ +from collections import ChainMap +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Final, cast # noqa: TID251 # live catalog has no typed boundary yet + +from pydantic import TypeAdapter + +from litellm.types.utils import ModelInfo + +_MODELS: Final = TypeAdapter(dict[str, ModelInfo]) +_DISCOUNTS: Final = TypeAdapter(dict[str, float]) +_MARGINS: Final = TypeAdapter(dict[str, float | dict[str, float]]) + + +@dataclass(frozen=True, slots=True) +class RetainedPricing: + models_json: str + discounts_json: str + margins_json: str + allow_live_models: bool = False + + def models(self) -> Mapping[str, ModelInfo]: + return _MODELS.validate_json(self.models_json) + + +_CURRENT: Final[ContextVar[RetainedPricing | None]] = ContextVar("retained_pricing", default=None) + + +@contextmanager +def retained_pricing(pricing: RetainedPricing) -> Generator[None]: + token: Final = _CURRENT.set(pricing) + try: + yield + finally: + _CURRENT.reset(token) + + +def retained_model_info(model: str, provider: str | None) -> ModelInfo | None: + context: Final = _CURRENT.get() + if context is None: + return None + models: Final = context.models() + result: Final = models.get(f"{provider}/{model}") or models.get(model) + if result is None and not context.allow_live_models: + raise ValueError(f"Pricing snapshot has no retained model {model}") + return result + + +def retained_models() -> Mapping[str, ModelInfo] | None: + import litellm + + context: Final = _CURRENT.get() + if context is None: + return None + if not context.allow_live_models: + return context.models() + catalog: Final = cast( # cast-ok: legacy catalog getter has an untyped dict signature + dict[str, ModelInfo], vars(litellm)["model_cost"] + ) + return ChainMap(dict(context.models()), catalog) + + +def pricing_discounts() -> Mapping[str, float]: + import litellm + + context: Final = _CURRENT.get() + return ( + litellm.cost_discount_config + if context is None or context.allow_live_models + else _DISCOUNTS.validate_json(context.discounts_json) + ) + + +def pricing_margins() -> Mapping[str, float | Mapping[str, float]]: + import litellm + + context: Final = _CURRENT.get() + return ( + litellm.cost_margin_config + if context is None or context.allow_live_models + else _MARGINS.validate_json(context.margins_json) + ) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 50c26f4d4ca..36a76811546 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -149,6 +149,19 @@ def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None +def image_response_size_and_quality( + response: ImageResponse, + optional_params: Mapping[str, object] | None = None, + *, + size: str | None = None, + quality: str | None = None, +) -> tuple[str, str]: + return ( + size or response.size or _requested_image_size(optional_params) or "1024-x-1024", + quality or response.quality or _requested_image_param(optional_params, "quality") or "standard", + ) + + def get_web_search_requests(server_tool_use: object) -> int | None: """ Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value @@ -2187,11 +2200,8 @@ class CostCalculatorUtils: cost_calculator as vertex_ai_image_cost_calculator, ) - resolved_size: Final = ( - size or completion_response.size or _requested_image_size(optional_params) or "1024-x-1024" - ) - resolved_quality: Final = ( - quality or completion_response.quality or _requested_image_param(optional_params, "quality") or "standard" + resolved_size, resolved_quality = image_response_size_and_quality( + completion_response, optional_params, size=size, quality=quality ) resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0) pricing: Final = deployment_pricing(model_info) diff --git a/litellm/main.py b/litellm/main.py index 78767b03eb7..d6706cdf99d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1316,7 +1316,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o ) from litellm.responses.bridge_check import ( bridge_reasoning_effort, - keep_registered_mode, str_keyed_mapping, ) from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site @@ -1472,50 +1471,26 @@ def _record_agentic_loop_params( ) -def _register_custom_pricing_for_request( +def _set_request_pricing( + logging_obj: object, model: str, custom_llm_provider: str, kwargs: dict, model_info: dict | None, ) -> None: - """Register per-request custom pricing in litellm.model_cost. - - Router-originated requests (identified by the deployment id the router puts - in metadata) get their full pricing registered under that unique id only; - the shared ``{provider}/{model}`` key receives the entry with pricing fields - stripped, mirroring Router._create_deployment. This keeps one deployment's - pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in - pricing used by sibling deployments of the same backend model. Direct SDK - calls keep the legacy behavior of registering the shared key with pricing. - Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on - deployment registration, since a call routes by its own ``model_info``. - """ - entry: Final = _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - shared_key: Final = f"{custom_llm_provider}/{model}" - deployment_id: Final = _get_router_deployment_id(kwargs) - if deployment_id is None: - litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False) + if not isinstance(logging_obj, LiteLLMLoggingObj): return - shared_entry: Final = keep_registered_mode( - shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry) + custom_pricing: Final = ( + (kwargs.get("input_cost_per_token") is not None and kwargs.get("output_cost_per_token") is not None) + or kwargs.get("input_cost_per_second") is not None + or kwargs.get("output_cost_per_second") is not None + or kwargs.get("cost_per_second") is not None ) - litellm.register_model( - {deployment_id: entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - litellm.register_model( - {shared_key: shared_entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, + logging_obj.set_request_pricing( + model, + custom_llm_provider, + _build_custom_pricing_entry(custom_llm_provider, kwargs, model_info) if custom_pricing else None, + _get_router_deployment_id(kwargs), ) @@ -5973,19 +5948,13 @@ def completion( "aws_region_name", None ) # support region-based pricing for bedrock - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) + _set_request_pricing( + logging_obj=litellm_logging_obj, + model=model, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=model_info, + ) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token: @@ -6992,12 +6961,6 @@ def embedding( headers.update(proxy_headers) except Exception as e: verbose_logger.warning("Failed to get proxy auth headers: %s", e) - ### CUSTOM MODEL COST ### - input_cost_per_token: Final = kwargs.get("input_cost_per_token", None) - output_cost_per_token: Final = kwargs.get("output_cost_per_token", None) - cost_per_second: Final = kwargs.get("cost_per_second", None) - input_cost_per_second: Final = kwargs.get("input_cost_per_second", None) - output_cost_per_second: Final = kwargs.get("output_cost_per_second", None) openai_params: Final = [ "user", "dimensions", @@ -7044,19 +7007,13 @@ def embedding( **non_default_params, ) - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=kwargs.get("model_info"), - ) + _set_request_pricing( + logging_obj=litellm_logging_obj, + model=model, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=kwargs.get("model_info"), + ) logging: Final[LiteLLMLoggingObj] = litellm_logging_obj logging.update_environment_variables( diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py index 0a9c5db1619..c321c21c06b 100644 --- a/litellm/proxy/db/accounting_ledger.py +++ b/litellm/proxy/db/accounting_ledger.py @@ -11,7 +11,17 @@ from typing import TYPE_CHECKING, Final, Literal, TypeAlias from pydantic import AwareDatetime, ConfigDict, JsonValue, TypeAdapter from typing_extensions import assert_never -from litellm.accounting import CostEvent, DeliveryTarget, EventKind, InboundUsageEvent, OutboundUsageEvent +from litellm.accounting import ( + CostEvent, + DeliveryTarget, + EventKind, + InboundUsageEvent, + Metric, + MetricName, + OutboundUsageEvent, + PricingSnapshot, + Unpriced, +) from litellm.proxy.db.db_span import db_span from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper @@ -143,6 +153,40 @@ SELECT records.event_id, destination.value::"AccountingDeliveryTarget", 1 FROM jsonb_to_recordset($1::jsonb) AS records(event_id text, targets jsonb) CROSS JOIN LATERAL jsonb_array_elements_text(records.targets) AS destination """ +_LOAD_SOURCE: Final = """ +SELECT payload::text AS payload, schema_version::text AS schema_version +FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 AND kind = 'outbound_usage' FOR UPDATE +""" +_LOAD_COST: Final = """ +SELECT payload::text AS payload, schema_version::text AS schema_version, payload_hash +FROM "LiteLLM_AccountingEvent" WHERE source_event_id = $1 AND kind = 'cost' +""" +_PENDING_COMPILATION: Final = """ +SELECT payload::text AS payload FROM "LiteLLM_AccountingEvent" +WHERE kind = 'outbound_usage' AND compilation_state = 'pending' +AND (compilation_retry_at IS NULL OR compilation_retry_at <= CURRENT_TIMESTAMP) +ORDER BY compilation_retry_at NULLS FIRST, recorded_at, event_id LIMIT $1::int +""" +_COMPILATION_RESULT: Final = """ +UPDATE "LiteLLM_AccountingEvent" SET compilation_state = $2, compilation_error = $3, compilation_retry_at = NULL +WHERE event_id = $1 AND kind = 'outbound_usage' +""" + + +_RETRY_COMPILATION: Final = """ +UPDATE "LiteLLM_AccountingEvent" +SET compilation_error = $2, compilation_retry_at = CURRENT_TIMESTAMP + INTERVAL '30 seconds' +WHERE event_id = $1 AND kind = 'outbound_usage' AND compilation_state = 'pending' +""" + + +async def _unpriced(db: Prisma, event_id: str, reason: str) -> Unpriced: + await db.execute_raw( + _COMPILATION_RESULT, event_id, "excluded" if reason == "not_charge_owner" else "unpriced", reason + ) + return Unpriced(usage_event_id=event_id, reason=reason) + + _COMPLETED_FAMILY: Final = """ root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage') AND root.recorded_at < $1::timestamp @@ -320,6 +364,79 @@ class AccountingLedger: case _: assert_never(result) + async def pending_compilation(self, limit: int = 100) -> tuple[OutboundUsageEvent, ...]: + if not 1 <= limit <= 1000: + raise ValueError("Compilation batch size must be between 1 and 1000") + async with self._transaction() as db: + rows: Final = _STRINGS.validate_python(await db.query_raw(_PENDING_COMPILATION, limit)) + return tuple( + OutboundUsageEvent.model_validate_json(_json(_JSON.validate_json(row["payload"])["event"])) for row in rows + ) + + async def defer_compilation(self, event: OutboundUsageEvent) -> None: + async with self._transaction() as db: + await db.execute_raw(_RETRY_COMPILATION, event.event_id, "compilation_retryable") + + async def compile_cost( + self, + event: OutboundUsageEvent, + price: Callable[[str], PricingSnapshot | Unpriced], + targets: tuple[DeliveryTarget, ...], + ) -> CostEvent | Unpriced: + """Serialize pure pricing with other compilers and commit its complete delivery set.""" + validated: Final = OutboundUsageEvent.model_validate_json(event.model_dump_json()) + async with self._transaction() as db: + sources: Final = _STRINGS.validate_python(await db.query_raw(_LOAD_SOURCE, validated.event_id)) + if not sources: + return Unpriced(usage_event_id=validated.event_id, reason="usage_not_committed") + source: Final = sources[0] + if source["schema_version"] != "1": + return await _unpriced(db, validated.event_id, "unsupported_usage_schema") + payload: Final = _JSON.validate_json(source["payload"]) + if payload.get("event") != validated.model_dump(mode="json"): + raise AccountingConflictError("Compilation input differs from the accepted usage event") + costs: Final = _STRINGS.validate_python(await db.query_raw(_LOAD_COST, validated.event_id)) + if costs: + if costs[0]["schema_version"] != "1": + return await _unpriced(db, validated.event_id, "unsupported_cost_schema") + existing: Final = CostEvent.model_validate_json( + _json(_JSON.validate_json(costs[0]["payload"])["event"]) + ) + if existing.usage_event_id != validated.event_id: + raise AccountingConflictError("Committed cost does not match its source usage") + replay: Final = _prepare(existing, targets, costs[0]["payload"], 1, 1) + if replay.payload_hash != costs[0]["payload_hash"]: + raise AccountingConflictError("Compilation delivery set differs from the accepted cost") + await db.execute_raw(_COMPILATION_RESULT, validated.event_id, "compiled", None) + return existing + try: + priced: Final = price(source["payload"]) + except ValueError: + return await _unpriced(db, validated.event_id, "invalid_pricing_inputs") + except Exception: # noqa: BLE001 # retain failed pricing for bounded retries without starving newer events + await db.execute_raw(_RETRY_COMPILATION, validated.event_id, "calculation_retryable") + return Unpriced(usage_event_id=validated.event_id, reason="calculation_retryable") + if isinstance(priced, Unpriced): + if priced.usage_event_id != validated.event_id: + raise AccountingConflictError("Unpriced result does not match its source usage") + return await _unpriced(db, validated.event_id, priced.reason) + cost: Final = CostEvent( + event_id=f"cost:{validated.event_id}", + usage_event_id=validated.event_id, + occurred_at=validated.ended_at, + dimensions=validated.dimensions, + charges=(Metric(name=MetricName.SPEND, value=priced.total_cost, unit="USD"),), + snapshot=priced, + ) + record: Final = _prepare( + cost, targets, _json({"event": cost.model_dump(mode="json"), "pricing_inputs": payload}), 1, 1 + ) + accepted: Final = await _accept(db, record) + if accepted not in ("inserted", "duplicate"): + raise AccountingConflictError(f"Cannot commit compiled cost: {accepted}") + await db.execute_raw(_COMPILATION_RESULT, validated.event_id, "compiled", None) + return cost + async def purge_completed( self, *, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c5fcbfb2564..5350d0a2adc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3256,11 +3256,15 @@ def _configure_usage_capture() -> None: "Usage capture requires local Prisma writes and disabled collector, remote writer and search free tier" ) from litellm.proxy.db.accounting_ledger import AccountingLedger + from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler ledger: Final = AccountingLedger.for_client(prisma_client) writer: Final = UsageBatchWriter(ledger.insert_usage_batch, ledger.load_usage_batch) usage_capture.start( - writer.write, UsageCaptureSettings.model_validate(capture_settings), load=ledger.load_usage_batch + writer.write, + UsageCaptureSettings.model_validate(capture_settings), + load=ledger.load_usage_batch, + compiler=LoggingCostCompiler(ledger), ) @@ -11607,6 +11611,22 @@ class ProxyStartupEvent: await cls._initialize_spend_tracking_background_jobs(scheduler=scheduler) + async def compile_accounting_costs() -> None: + from litellm.proxy.db.accounting_ledger import AccountingLedger + from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler + + if usage_capture.meter is not None: + await LoggingCostCompiler(AccountingLedger.for_client(prisma_client)).compile_pending() + + scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # APScheduler has no typed add_job interface + compile_accounting_costs, + "interval", + seconds=5, + id="accounting_cost_compilation", + replace_existing=True, + max_instances=1, + ) + cls._initialize_spend_capture_rate_check_job( scheduler=scheduler, proxy_logging_obj=proxy_logging_obj, diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 179b741fe7c..c2f34e3b17e 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -2111,8 +2111,12 @@ model LiteLLM_AccountingEvent { occurred_at DateTime recorded_at DateTime @default(now()) payload Json + compilation_state String @default("pending") + compilation_error String? + compilation_retry_at DateTime? @@index([recorded_at]) + @@index([kind, compilation_state, recorded_at]) @@index([kind, source_event_id]) } diff --git a/litellm/proxy/spend_tracking/cost_compiler.py b/litellm/proxy/spend_tracking/cost_compiler.py new file mode 100644 index 00000000000..1afa83af83f --- /dev/null +++ b/litellm/proxy/spend_tracking/cost_compiler.py @@ -0,0 +1,463 @@ +import json +import math +from collections.abc import Callable, Mapping +from datetime import datetime +from decimal import Decimal +from typing import Final + +from httpx import Response +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +from litellm._internal_context import pinned_billing_time +from litellm._logging import verbose_proxy_logger +from litellm.accounting import ( + CostEvent, + DeliveryTarget, + DimensionName, + OutboundUsageEvent, + Outcome, + PricingSnapshot, + Unpriced, +) +from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost +from litellm.litellm_core_utils.hidden_params import set_hidden_param +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.llm_cost_calc.finalization import finalize_cost +from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing +from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo +from litellm.proxy.db.accounting_ledger import AccountingLedger +from litellm.proxy.spend_tracking.usage_meter import usage_delivery_targets +from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse +from litellm.types.rerank import RerankResponse +from litellm.types.utils import ( + CallTypes, + CostBreakdown, + CostPerToken, + ImageResponse, + LiteLLMRealtimeStreamLoggingObject, + ModelInfo, + ModelResponse, + StandardBuiltInToolsParams, + Usage, + is_server_derived_pricing_key, +) + +_FIELDS: Final = TypeAdapter(dict[str, JsonValue]) + + +def _fields(value: object) -> Mapping[str, JsonValue]: + return _FIELDS.validate_python(value) if isinstance(value, dict) else {} + + +def _json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +class _UsageEnvelope(BaseModel): + model_config = ConfigDict(frozen=True) + + event: OutboundUsageEvent + usage: Mapping[str, JsonValue] | None = None + pricing: Mapping[str, JsonValue] + charge_eligible: bool + parent_request_id: str | None = None + finalized_cost: Mapping[str, JsonValue] | None = None + allocation: Mapping[str, JsonValue] | None = None + + +def price_usage(envelope: str) -> PricingSnapshot | Unpriced: + captured: Final = _UsageEnvelope.model_validate_json(envelope) + event: Final = captured.event + batch_summary: Final = event.call_type in {CallTypes.retrieve_batch, CallTypes.aretrieve_batch} and not ( + captured.parent_request_id is not None and event.event_id.startswith(f"{captured.parent_request_id}:row:") + ) + if not captured.charge_eligible or batch_summary: + return Unpriced(usage_event_id=event.event_id, reason="not_charge_owner") + response_inputs: Final = _fields(captured.pricing.get("response_inputs")) + rerank_usage: Final = event.call_type in {CallTypes.rerank, CallTypes.arerank} and any( + isinstance(value, int) and not isinstance(value, bool) and value >= 0 + for name, value in _fields(response_inputs.get("billed_units")).items() + if name in {"search_units", "total_tokens"} + ) + logged: Final = _fields(captured.pricing.get("logging")) + speech_characters: Final = logged.get("prompt_characters") + speech_usage: Final = ( + event.call_type in {CallTypes.speech, CallTypes.aspeech} + and isinstance(speech_characters, int) + and not isinstance(speech_characters, bool) + and speech_characters >= 0 + ) + fixed_call: Final = event.call_type in {CallTypes.call_mcp_tool, CallTypes.send_message, CallTypes.asend_message} + realtime_events: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python( + captured.pricing.get("realtime_events") or () + ) + realtime_usage: Final = event.call_type is CallTypes.arealtime and any( + bool(item.get("usage")) for item in realtime_events + ) + image_inputs: Final = _fields(captured.pricing.get("image_response")) + if ( + captured.usage is None + and not fixed_call + and not realtime_usage + and not speech_usage + and not rerank_usage + and not image_inputs + ): + return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") + pricing: Final = captured.pricing + mapped: Final = _fields(pricing.get("model_map_information")) + rates: Final = _fields(mapped.get("model_map_value")) + if ( + event.call_type in {CallTypes.speech, CallTypes.aspeech} + and rates.get("input_cost_per_character") is not None + and not speech_usage + ): + return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") + model: Final = next( + (d.value for d in event.dimensions if d.name is DimensionName.MODEL), + event.call_type.value if fixed_call else None, + ) + provider: Final = next((d.value for d in event.dimensions if d.name is DimensionName.PROVIDER), None) + if ( + fixed_call + and event.outcome is not Outcome.SUCCEEDED + and captured.usage is None + and not logged.get("tool_response_cost") + ): + return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") + reported: Final = ( + logged.get("provider_reported_cost") + if logged.get("provider_reported_cost") is not None + else logged.get("provider_usage_cost") + ) + upstream: Final = ( + isinstance(reported, (int, float)) + and not isinstance(reported, bool) + and math.isfinite(reported) + and reported >= 0 + and logged.get("custom_pricing") is not True + ) + key: Final = mapped.get("model_map_key") or (model if fixed_call or upstream else None) + has_rates: Final = any(is_server_derived_pricing_key(name) and value is not None for name, value in rates.items()) + if model is None or not isinstance(key, str) or (not has_rates and not fixed_call and not upstream): + return Unpriced(usage_event_id=event.event_id, reason="pricing_inputs_missing") + info: Final = { + "key": key, + "litellm_provider": provider, + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": None, + "output_cost_per_token": None, + "supported_openai_params": None, + **rates, + "mode": rates.get("mode") or "chat", + } + adjustments: Final = _fields(pricing.get("adjustments")) + discount: Final = adjustments.get("discount") + margin: Final = adjustments.get("margin") + secondary: Final = { + name: { + **info, + **_fields(_fields(snapshot).get("model_map_value")), + "key": _fields(snapshot).get("model_map_key") or name, + } + for name, snapshot in {**_fields(pricing.get("realtime_models")), **_fields(pricing.get("tool_models"))}.items() + if _fields(snapshot).get("model_map_value") is not None + } + base_pricing: Final = _fields(pricing.get("base_model_information")) + base_key: Final = base_pricing.get("model_map_key") + context: Final = RetainedPricing( + _json( + { + **secondary, + **( + {base_key: _fields(base_pricing.get("model_map_value"))} + if isinstance(base_key, str) and base_pricing.get("model_map_value") is not None + else {} + ), + **{ + alias: info + for alias in dict.fromkeys((model, key, f"{provider}/{model}", model.removeprefix(f"{provider}/"))) + }, + } + ), + _json({provider: discount} if provider and discount is not None else {}), + _json({"global": margin} if margin is not None else {}), + ) + tool_response: Final = _fields(pricing.get("tool_response")) + replay_usage: Final = Usage.model_validate( + { + **{name: value for name, value in (captured.usage or {}).items() if name != "cost"}, + **({"cost": reported} if upstream and logged.get("provider_reported_cost") is None else {}), + } + ) + token_response: Final = ModelResponse( + model=model, + choices=[ + { + "message": { + "role": "assistant", + "annotations": tuple( + {"type": annotation} + for flag, annotation in (("web_search", "url_citation"),) + if tool_response.get(flag) + ), + } + } + ], + usage=replay_usage, + service_tier=response_inputs.get("service_tier"), + meta={"billed_units": response_inputs.get("billed_units")}, + hidden_params={ + "region_name": response_inputs.get("region_name"), + "audio_transcription_duration": response_inputs.get("audio_transcription_duration"), + }, + response_ms=logged.get("response_ms"), + ) + speech_response: Final = HttpxBinaryResponseContent(Response(200, content=b"")) if speech_usage else None + if speech_response is not None: + set_hidden_param(speech_response, "billed_characters", speech_characters) + speech_response.usage = replay_usage + response: Final = ( + speech_response + if speech_response is not None + else LiteLLMRealtimeStreamLoggingObject.model_validate( + { + "results": pricing.get("realtime_events", []), + "usage": captured.usage or {}, + "service_tier": response_inputs.get("service_tier"), + } + ) + if event.call_type in {CallTypes.arealtime, CallTypes.aresponses_websocket} + else RerankResponse.model_validate( + {"id": event.event_id, "results": [], "meta": {"billed_units": response_inputs.get("billed_units")}} + ) + if event.call_type in {CallTypes.rerank, CallTypes.arerank} + else ResponsesAPIResponse.model_validate( + { + "id": event.event_id, + "created_at": int(event.started_at.timestamp()), + "model": model, + "output": tool_response.get("output", []), + "usage": tool_response.get("usage"), + "service_tier": response_inputs.get("service_tier"), + "tools": tool_response.get("tools"), + "tool_usage": tool_response.get("tool_usage"), + } + ) + if tool_response.get("responses_api") + else ImageResponse.model_validate( + {"created": int(event.started_at.timestamp()), "model": model, **image_inputs} + ) + if image_inputs + else token_response + ) + logger: Final = Logging( + model=model, + messages=[], + stream=False, + call_type=CallTypes.aresponses.value + if event.call_type in {CallTypes.get_responses, CallTypes.aget_responses} + else event.call_type.value, + start_time=event.started_at, + litellm_call_id=event.event_id, + function_id=event.attempt_id, + ) + request: Final = _fields(pricing.get("request")) + deployment: Final = _fields(pricing.get("deployment")) + logger.update_environment_variables( # pyright: ignore[reportUnknownMemberType] # legacy Logging environment method accepts validated JSON fields + model=model, + custom_llm_provider=provider, + litellm_params={**deployment, **request, "metadata": {"model_info": _fields(pricing.get("model_info"))}}, + additional_response_cost=request.get("additional_response_cost"), + usage=replay_usage, + end_time=event.ended_at, + mcp_tool_call_metadata={ + "name": _fields(pricing.get("tool")).get("name"), + "mcp_server_cost_info": {"default_cost_per_query": _fields(pricing.get("tool")).get("cost_per_query")}, + }, + optional_params=dict(_fields(pricing.get("optional_params"))), + ) + if logged.get("tool_response_cost") is not None: + logger.model_call_details["response_cost"] = logged["tool_response_cost"] + logger.request_pricing = context if logged.get("custom_pricing") is True else None + logger.custom_cost_per_token = TypeAdapter(CostPerToken | None).validate_python(logged.get("custom_cost_per_token")) + logger.custom_cost_per_second = TypeAdapter(float | None).validate_python(logged.get("custom_cost_per_second")) + logger.standard_built_in_tools_params = TypeAdapter(StandardBuiltInToolsParams).validate_python( + { + "web_search_options": adjustments.get("web_search_options"), + "file_search": adjustments.get("file_search") or None, + } + ) + if upstream and logged.get("provider_reported_cost") is not None: + set_hidden_param(response, "additional_headers", {"llm_provider-x-litellm-response-cost": reported}) + billed_at: Final = TypeAdapter(datetime).validate_python(logged.get("current_billing_time", event.ended_at)) + if event.call_type is CallTypes.arealtime and captured.usage and captured.usage.get("metered_usage") is not None: + from litellm.constants import REALTIME_METERED_USAGE_KEY + from litellm.types.llms.xai import RealtimeMeteredUsage + + logger.model_call_details[REALTIME_METERED_USAGE_KEY] = TypeAdapter(RealtimeMeteredUsage).validate_python( + captured.usage["metered_usage"] + ) + logger.cost_billing_time = billed_at + with retained_pricing(context), pinned_billing_time(billed_at): + batch_costs: Final = ( + ocr_batch_cost( + model=model, + custom_llm_provider=provider, + usage_info=OCRUsageInfo.model_validate(captured.usage), + model_info=TypeAdapter(ModelInfo).validate_python(info), + ) + if event.call_type in {CallTypes.retrieve_batch, CallTypes.aretrieve_batch} + and any(name in (captured.usage or {}) for name in OCRUsageInfo.model_fields) + else batch_cost_calculator( + replay_usage, + model, + provider, + data_residency=TypeAdapter[str | None](str | None).validate_python(deployment.get("data_residency")), + vertex_location=TypeAdapter[str | None](str | None).validate_python(deployment.get("vertex_location")), + ) + if event.call_type in {CallTypes.retrieve_batch, CallTypes.aretrieve_batch} + else None + ) + amount: Final = ( + sum(batch_costs) + if batch_costs is not None + else logger.response_cost_calculator( + response, + ignore_stamped_cost=True, + custom_pricing_override=TypeAdapter[bool | None](bool | None).validate_python( + logged.get("custom_pricing") + ), + ) + ) + if batch_costs is not None: + logger.cost_breakdown = CostBreakdown( + input_cost=batch_costs[0], output_cost=batch_costs[1], total_cost=sum(batch_costs) + ) + if amount is None: + return Unpriced(usage_event_id=event.event_id, reason="calculation_failed") + retained_final: Final = captured.finalized_cost or {} + breakdown: Final = _fields(retained_final.get("cost_breakdown")) + guardrail: Final = TypeAdapter(float).validate_python( + breakdown.get("guardrail_cost", pricing.get("guardrail_cost", 0.0)) + ) + allowance: Final = TypeAdapter(float).validate_python(retained_final.get("allowance_discount", 0.0)) + if allowance and captured.allocation is None: + return Unpriced(usage_event_id=event.event_id, reason="allowance_allocation_missing") + finalized: Final = finalize_cost(amount + guardrail, logger.cost_breakdown, allowance) + if retained_final.get("response_cost") is not None and Decimal(str(finalized.amount)) != Decimal( + str(retained_final["response_cost"]) + ): + return Unpriced(usage_event_id=event.event_id, reason="finalized_cost_mismatch") + final_breakdown: Final = dict(finalized.breakdown or {}) + components: Final = { + name: Decimal(str(value)) + for name, value in final_breakdown.items() + if name in _COST_COMPONENTS and isinstance(value, (int, float)) + } + return PricingSnapshot.model_validate( + { + **components, + "calculator_version": "logging-v1", + "model": model, + "provider": provider, + "pricing_source": "upstream_header" + if upstream + else "per_request" + if any(is_server_derived_pricing_key(name) and value is not None for name, value in request.items()) + else "deployment" + if any(is_server_derived_pricing_key(name) and value is not None for name, value in deployment.items()) + else "cost_map", + "priced_model_key": key, + "total_cost": Decimal(str(finalized.amount)), + "guardrail_cost": Decimal(str(guardrail)), + "upstream_reported_cost": Decimal(str(reported)) if upstream else None, + "additional_costs": tuple( + {"name": name, "amount": Decimal(str(value))} + for name, value in { + **_fields(final_breakdown.get("additional_costs")), + **( + {"request": request["additional_response_cost"]} + if request.get("additional_response_cost") + else {} + ), + }.items() + ), + } + ) + + +_COST_COMPONENTS: Final = frozenset( + { + "input_cost", + "output_cost", + "cache_read_cost", + "cache_creation_cost", + "reasoning_cost", + "tool_usage_cost", + "original_cost", + "discount_percent", + "discount_amount", + "margin_percent", + "margin_fixed_amount", + "margin_total_amount", + } +) + + +class LoggingCostCompiler: + def __init__( + self, + ledger: AccountingLedger, + price_usage: Callable[[str], PricingSnapshot | Unpriced] = price_usage, + delivery_targets: tuple[DeliveryTarget, ...] | None = None, + ) -> None: + self._ledger: Final = ledger + self._price_usage: Final = price_usage + self._delivery_targets: Final = delivery_targets + + async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced: + targets: Final = self._delivery_targets if self._delivery_targets is not None else cost_delivery_targets(event) + return await self._ledger.compile_cost(event, self._price_usage, targets) + + async def compile_pending(self, limit: int = 100) -> None: + for event in await self._ledger.pending_compilation(limit): + try: + await self.compile(event) + except Exception: # noqa: BLE001 # retain pending work for retry without starving other events + verbose_proxy_logger.exception("Cost compilation failed for %s", event.event_id) + try: + await self._ledger.defer_compilation(event) + except Exception: # noqa: BLE001 # database outages must not prevent attempts on later events + verbose_proxy_logger.exception( + "Cost compilation retry could not be scheduled for %s", event.event_id + ) + + +_SPEND_TARGETS: Final = ( + (DimensionName.KEY, DeliveryTarget.KEY_SPEND), + (DimensionName.USER, DeliveryTarget.USER_SPEND), + (DimensionName.TEAM, DeliveryTarget.TEAM_SPEND), + (DimensionName.ORGANIZATION, DeliveryTarget.ORGANIZATION_SPEND), + (DimensionName.END_USER, DeliveryTarget.END_USER_SPEND), + (DimensionName.AGENT, DeliveryTarget.AGENT_SPEND), + (DimensionName.TAG, DeliveryTarget.TAG_SPEND), + (DimensionName.PROJECT, DeliveryTarget.PROJECT_SPEND), + (DimensionName.MODEL_ACCESS_GROUP, DeliveryTarget.MODEL_ACCESS_GROUP_SPEND), +) + + +def cost_delivery_targets(event: OutboundUsageEvent) -> tuple[DeliveryTarget, ...]: + names: Final = frozenset(d.name for d in event.dimensions) + return ( + *usage_delivery_targets(event.dimensions), + *(target for dimension, target in _SPEND_TARGETS if dimension in names), + *((DeliveryTarget.TEAM_MEMBER_SPEND,) if {DimensionName.TEAM, DimensionName.USER} <= names else ()), + *( + (DeliveryTarget.ORGANIZATION_MEMBER_SPEND,) + if {DimensionName.ORGANIZATION, DimensionName.USER} <= names + else () + ), + ) diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index 19480a66678..22ea090629a 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -8,13 +8,14 @@ from dataclasses import dataclass, replace from datetime import datetime, timezone from decimal import Decimal from hashlib import sha256 -from itertools import chain, groupby, islice, repeat +from itertools import chain, groupby, islice, product, repeat from typing import TYPE_CHECKING, Final from pydantic import BaseModel, JsonValue, TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.accounting import ( + CostCompiler, Dimension, DimensionName, InboundUsageEvent, @@ -30,7 +31,7 @@ from litellm.proxy.spend_tracking.usage_meter import ( UsageCaptureSettings, usage_delivery_targets, ) -from litellm.types.utils import CallTypes, CustomPricingLiteLLMParams +from litellm.types.utils import CallTypes, CustomPricingLiteLLMParams, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging @@ -172,7 +173,15 @@ def usage_writes( ended_at=ended, outcome=outcome, dimensions=dimensions, - metrics=_usage_metrics(None if context.usage_source == "unknown" else usage), + metrics=_usage_metrics( + None + if context.usage_source == "unknown" + or ( + context.ownership == "aggregate" + and payload["call_type"] in {CallTypes.retrieve_batch.value, CallTypes.aretrieve_batch.value} + ) + else usage + ), ) envelope: Final = { "event": outbound.model_dump(mode="json"), @@ -217,6 +226,7 @@ def usage_writes( _INCLUDED_IN_PARENT: Final = object() +_BATCH_ROWS: Final = object() _PRICING_FIELDS: Final = frozenset( @@ -240,6 +250,7 @@ _PRICING_FIELDS: Final = frozenset( "seconds", "search_context_size", "additional_response_cost", + "cost_per_query", } ) @@ -279,6 +290,7 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res mask_api_base_credentials, ) from litellm.litellm_core_utils.llm_cost_calc.finalization import FinalizedCost + from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation from litellm.proxy.spend_tracking.spend_tracking_utils import get_request_model_access_groups @@ -303,9 +315,14 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res "expired", "incomplete", } + batch_summary: Final = ( + payload["call_type"] in {CallTypes.retrieve_batch.value, CallTypes.aretrieve_batch.value} + and kwargs.get("_accounting_batch_rows") is not _BATCH_ROWS + ) deferred: Final = billed_id.endswith(("_batch_cost", "_background_cost")) and terminal management: Final = ( - (billed_id.endswith(("_batch_cost", "_background_cost")) and not terminal) + payload["call_type"] in {CallTypes.create_batch.value, CallTypes.acreate_batch.value} + or (billed_id.endswith(("_batch_cost", "_background_cost")) and not terminal) or (is_background_response(response) and not terminal) or is_unbilled_non_inference_call(payload["call_type"], request_metadata, response) ) @@ -316,6 +333,8 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res if management else f"{billed_id}:included" if included_in_parent + else f"{billed_id}:summary" + if batch_summary and terminal else billed_id ) if not attempt or attempt == "None" or (cached and not caller): @@ -333,7 +352,11 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res else kwargs.get("_accounting_allocation") ) allocation: Final = allocation_value if isinstance(allocation_value, SearchAllowanceAllocation) else None - pricing_snapshot: Final = kwargs.get("_accounting_pricing_snapshot") + pricing_snapshot: Final = ( + _capture_pricing(logging_obj, response, kwargs) + if isinstance(logging_obj, Logging) and _batch_rows(response) + else kwargs.get("_accounting_pricing_snapshot") + ) observation: Final = kwargs.get("_accounting_usage_snapshot") captured_pricing: Final = { "model_map_information": ( @@ -342,6 +365,7 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res else metadata.get("model_map_information") ), "cost_breakdown": metadata.get("cost_breakdown"), + "guardrail_cost": guardrail_information_cost(metadata.get("guardrail_information")), **( _JSON_OBJECT.validate_json(pricing_snapshot.inputs) if isinstance(pricing_snapshot, PricingCapture) @@ -381,12 +405,12 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res parent_request_id=parent, retry_of_attempt_id=_text(request_metadata.get("retry_of_attempt_id")), job_id=_text(raw_response.get("id")) if deferred or management else None, - charge_eligible=not cached and not included_in_parent and not management, + charge_eligible=not cached and not included_in_parent and not management and not batch_summary, request_count_eligible=bool(caller) and not origin and not retry and not deferred and not included_in_parent, ownership="included_in_parent" if included_in_parent else "aggregate" - if kwargs.get("additional_response_cost") is not None + if batch_summary or kwargs.get("additional_response_cost") is not None else "attempt", pricing=_json(captured_pricing), finalized_cost=_json( @@ -555,13 +579,26 @@ def _batch_rows(response: object) -> tuple[Mapping[str, JsonValue], ...]: def _batch_root(write: UsageWrite, rows: tuple[Mapping[str, JsonValue], ...]) -> UsageWrite: - event: Final = write.event.model_copy(update={"metrics": None}) + from litellm.proxy.openai_files_endpoints.common_utils import ( + convert_b64_uid_to_unified_uid, + get_batch_id_from_unified_batch_id, + get_original_file_id, + ) + envelope: Final = _JSON_OBJECT.validate_json(write.envelope) + observed_job: Final = TypeAdapter(str).validate_python(envelope["job_id"]) + decoded: Final = convert_b64_uid_to_unified_uid(observed_job) + job: Final = ( + get_batch_id_from_unified_batch_id(decoded) if decoded != observed_job else get_original_file_id(observed_job) + ) + identity: Final = f"{job}_batch_cost" + event: Final = write.event.model_copy(update={"metrics": None, "event_id": identity, "attempt_id": identity}) return _database_usage_write( event, { **envelope, "event": event.model_dump(mode="json"), + "job_id": job, "usage": { "batch_row_count": len(rows), "batch_rows_sha256": sha256(_json(rows).encode()).hexdigest(), @@ -635,7 +672,7 @@ def _prepare_batch_capture( from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload rows: Final = tuple(sorted(_batch_rows(response), key=_json)) - builder: Final = _CaptureBuilder(kwargs, response) + builder: Final = _CaptureBuilder({**kwargs, "_accounting_batch_rows": _BATCH_ROWS}, response) payload: Final = get_logging_payload( dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe ) @@ -782,10 +819,27 @@ async def capture_proxy_response(logging_obj: object, response: object, *, backg timeout: Final = TypeAdapter(float).validate_python( settings.get("accounting_sync_ack_timeout_seconds", 20.0) ) - await asyncio.wait_for(asyncio.shield(captured.receipt), timeout=timeout) + async with asyncio.timeout(timeout): + await asyncio.shield(captured.receipt) + compiler: Final = usage_capture.compiler + if compiler is not None: + for write in captured.writes: + if isinstance(write.event, OutboundUsageEvent): + await compiler.compile(write.event) except Exception as error: # noqa: BLE001 # the configured response policy owns every capture failure verbose_proxy_logger.exception("Usage capture failed before the response boundary") - if sync_ack and settings.get("accounting_sync_ack_failure_policy", "fail_closed") == "fail_closed": + accepted: Final = logging_obj.model_call_details.get("_accounting_capture") + committed: Final = ( + isinstance(accepted, CapturedUsage) + and accepted.receipt.done() + and not accepted.receipt.cancelled() + and accepted.receipt.exception() is None + ) + if ( + sync_ack + and not committed + and settings.get("accounting_sync_ack_failure_policy", "fail_closed") == "fail_closed" + ): acknowledgement: Final = UsageAcknowledgementError(error) raise HTTPException(503, str(acknowledgement)) from acknowledgement @@ -828,10 +882,10 @@ def capture_logging_usage( existing_pricing: Final = details.get("_accounting_pricing_snapshot") pricing: Final = ( existing_pricing - if isinstance(existing_pricing, PricingCapture) and not internal - else _capture_pricing(logging_obj, normalized, details) + if isinstance(existing_pricing, PricingCapture) and not internal and not _batch_rows(normalized) + else _capture_pricing(logging_obj, normalized_result, details) ) - if freeze_only or _batch_rows(normalized): + if freeze_only or logging_obj.call_type in {CallTypes.retrieve_batch.value, CallTypes.aretrieve_batch.value}: if not internal: logging_obj.record_usage_pricing(pricing) return @@ -866,9 +920,21 @@ class PricingCapture: def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[str, object]) -> PricingCapture: + from litellm.litellm_core_utils.llm_cost_calc.pricing_context import retained_pricing + + if logging_obj.request_pricing is not None and not logging_obj.spilled_over_ptu_for(normalized): + with retained_pricing(logging_obj.request_pricing): + return _capture_effective_pricing(logging_obj, normalized, details) + return _capture_effective_pricing(logging_obj, normalized, details) + + +def _capture_effective_pricing( + logging_obj: Logging, normalized: object, details: Mapping[str, object] +) -> PricingCapture: import litellm from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.utils import get_base_model_from_metadata params: Final = _fields(details.get("litellm_params")) prepared: Final = logging_obj.prepared_cost @@ -879,7 +945,7 @@ def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[ if prepared is not None else StandardLoggingPayloadSetup.get_model_cost_information( base_model=logging_obj.get_deployment_model_for_cost(), - custom_pricing=bool(params.get("custom_pricing")), + custom_pricing=logging_obj.custom_pricing_for(normalized), custom_llm_provider=_text(details.get("custom_llm_provider")), init_response_obj=normalized, api_base=_text(params.get("api_base")), @@ -900,16 +966,62 @@ def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[ if builtins.get("web_search_options") is not None else None ), - "file_search": builtins.get("file_search") is not None, + "file_search": { + key: value + for key, value in _fields(builtins.get("file_search")).items() + if key in {"type", "storage_gb", "days"} + } + if builtins.get("file_search") is not None + else None, } ) - batch_usage: Final = _fields(_fields(_response_fields(normalized).get("usage")).get("batch_usage_by_model")) + base_model: Final = get_base_model_from_metadata(model_call_details=logging_obj.model_call_details) + realtime_models: Final = tuple( + dict.fromkeys( + chain.from_iterable(_realtime_pricing_models(event) for event in _realtime_cost_events(normalized)) + ) + ) + batch_usage: Final = _fields( + _response_fields(_response_fields(normalized).get("usage")).get("batch_usage_by_model") + ) return PricingCapture( _json(pricing), adjustments, _json( { **_pricing_inputs(details, logging_obj, normalized), + "base_model_information": StandardLoggingPayloadSetup.get_model_cost_information( + base_model=base_model, + custom_pricing=False, + custom_llm_provider=None, + init_response_obj={"model": base_model}, + ) + if base_model is not None + else None, + "realtime_models": { + model: StandardLoggingPayloadSetup.get_model_cost_information( + base_model=model, + custom_pricing=False, + custom_llm_provider=provider, + init_response_obj={"model": model}, + ) + for model in realtime_models + }, + "tool_models": { + model: StandardLoggingPayloadSetup.get_model_cost_information( + base_model=model, + custom_pricing=False, + custom_llm_provider=provider, + init_response_obj={"model": model}, + ) + for model in _image_pricing_models( + normalized, + provider, + logging_obj.get_deployment_model_for_cost(), + _fields(details.get("optional_params")), + ) + if model in _fields(vars(litellm)["model_cost"]) + }, "batch_models": { model: StandardLoggingPayloadSetup.get_model_cost_information( base_model=model, @@ -926,10 +1038,168 @@ def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[ ) +def _tool_cost_inputs(response: object) -> Mapping[str, object]: + from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import StandardBuiltInToolCostTracking + from litellm.types.llms.openai import ResponsesAPIResponse + + raw: Final = _response_fields(response) + flags: Final = { + "web_search": StandardBuiltInToolCostTracking.response_object_includes_web_search_call(response), + "file_search": StandardBuiltInToolCostTracking.response_object_includes_file_search_call(response), + } + if not isinstance(response, ResponsesAPIResponse) and raw.get("object") != "response": + return flags if any(flags.values()) else {} + return { + **flags, + "responses_api": True, + "tools": tuple( + {key: value for key, value in _fields(tool).items() if key in {"type", "model"}} + for tool in TypeAdapter(tuple[object, ...]).validate_python(raw.get("tools") or ()) + if _fields(tool).get("type") == "image_generation" + ), + "tool_usage": _response_fields(raw.get("tool_usage")) or None, + "usage": _response_fields(raw.get("usage")), + "output": tuple( + { + key: value + for key, value in _fields(item).items() + if key in {"type", "id", "status", "usage", "quality", "size"} + } + for item in TypeAdapter(tuple[object, ...]).validate_python(raw.get("output") or ()) + ), + } + + +def _image_pricing_models( + response: object, provider: str | None, deployment_model: str | None, optional_params: Mapping[str, object] +) -> tuple[str, ...]: + from litellm.cost_calculator import image_cost_model_candidates + from litellm.litellm_core_utils.llm_cost_calc.utils import image_response_size_and_quality + + if isinstance(response, ImageResponse) and deployment_model is not None: + size, quality = image_response_size_and_quality(response, optional_params) + return tuple( + key + for key in image_cost_model_candidates( + deployment_model, + provider, + quality, + size, + ) + if key is not None + ) + raw: Final = _tool_cost_inputs(response) + models: Final = tuple( + model + for tool in TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(raw.get("tools") or ()) + if isinstance(model := tool.get("model"), str) + ) + items: Final = tuple( + item + for item in TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(raw.get("output") or ()) + if item.get("type") == "image_generation_call" and item.get("status") == "completed" + ) + candidates: Final = chain.from_iterable( + image_cost_model_candidates( + model, + provider, + _text(item.get("quality")) if item.get("quality") != "auto" else None, + _text(item.get("size")) if item.get("size") != "auto" else None, + ) + for model, item in product(models, items) + ) + return tuple(dict.fromkeys(model for model in candidates if model is not None)) + + +_REALTIME_CHILDREN: Final = ( + "response", + "event", + "session", + "audio", + "input", + "transcription", + "input_audio_transcription", +) + + +def _realtime_cost_node( + fields: Mapping[str, object], +) -> dict[str, JsonValue]: # mutable-ok: tree builder attaches sanitized children + return { + **{ + key: _JSON_OBJECT.validate_python(fields[key]) for key in ("usage",) if isinstance(fields.get(key), Mapping) + }, + **{ + key: value + for key, value in fields.items() + if key in {"type", "id", "model", "service_tier"} and isinstance(value, str) + }, + } + + +def _realtime_cost_fields(value: object) -> Mapping[str, JsonValue]: + root: Final = _realtime_cost_node(_fields(value)) + pending: Final[list[tuple[Mapping[str, object], dict[str, JsonValue]]]] = [ # mutable-ok: traversal stack + (_fields(value), root) + ] + while pending: + source, target = pending.pop() + for key in _REALTIME_CHILDREN: + if not isinstance(source.get(key), Mapping): + continue + fields: Final = _fields(source[key]) # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value + child: Final = _realtime_cost_node(fields) # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value + target[key] = child + pending.append((fields, child)) + return root + + +def _realtime_cost_events(response: object) -> tuple[Mapping[str, JsonValue], ...]: + results: Final = _response_fields(response).get("results") + return ( + tuple(_realtime_cost_fields(result) for result in TypeAdapter(tuple[object, ...]).validate_python(results)) + if isinstance(results, list) + else () + ) + + +def _realtime_pricing_models(value: Mapping[str, JsonValue]) -> Iterator[str]: + pending: Final[list[Mapping[str, JsonValue]]] = [ # mutable-ok: iterative traversal avoids recursion + value + ] + while pending: + fields: Final = pending.pop() # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value + model: Final = fields.get("model") # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value + if isinstance(model, str): + yield model + pending.extend(nested for key in _REALTIME_CHILDREN if isinstance(nested := fields.get(key), dict)) + + +def _image_cost_dimensions(image: object) -> Mapping[str, object]: + fields: Final = _fields(_response_fields(image).get("provider_specific_fields")) + return { + "provider_specific_fields": { + name: value + for name, value in fields.items() + if name in {"width", "height"} and isinstance(value, int) and not isinstance(value, bool) + } + } + + def _pricing_inputs(kwargs: Mapping[str, object], logging_obj: object, response: object) -> Mapping[str, object]: from litellm.litellm_core_utils.litellm_logging import Logging return { + "image_response": { + "data": tuple(_image_cost_dimensions(image) for image in response.data or ()), + "usage": _response_fields(response.usage) if response.usage is not None else None, + "size": response.size, + "quality": response.quality, + } + if isinstance(response, ImageResponse) + else None, + "tool_response": _tool_cost_inputs(response), + "realtime_events": _realtime_cost_events(response), "request": _selected_pricing(kwargs), "deployment": _selected_pricing(_fields(kwargs.get("litellm_params"))), "optional_params": _selected_pricing(_fields(kwargs.get("optional_params"))), @@ -1006,6 +1276,7 @@ class UsageCaptureRuntime: self._batches: Final[set[asyncio.Task[None]]] = set() # mutable-ok: owned producer lifecycle self._stopping = False self.meter: DurableUsageMeter | None = None + self.compiler: CostCompiler | None = None self.load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None def start( @@ -1014,6 +1285,7 @@ class UsageCaptureRuntime: settings: UsageCaptureSettings, *, load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None, + compiler: CostCompiler | None = None, ) -> None: from litellm.litellm_core_utils.litellm_logging import Logging @@ -1024,6 +1296,7 @@ class UsageCaptureRuntime: meter: Final = DurableUsageMeter(write_batch, settings) self.meter = meter self.load = load + self.compiler = compiler self._stopping = False Logging.usage_capture_callback = lambda logger, response, start, end, freeze: capture_logging_usage( meter, logger, response, start, end, freeze @@ -1082,3 +1355,4 @@ class UsageCaptureRuntime: Logging.usage_capture_callback = None self.meter = None self.load = None + self.compiler = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index da1d99957a7..d66cd613503 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -437,6 +437,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_token_above_512k_tokens: float | None # MiniMax-M3: prompts >512K priced at 2x output output_cost_per_character_above_128k_tokens: float | None # only for vertex ai models output_cost_per_image: float | None + input_cost_per_pixel: ReadOnly[float | None] output_cost_per_pixel: ReadOnly[float | None] output_cost_per_image_token: float | None output_cost_per_image_token_batches: ReadOnly[float | None] @@ -488,6 +489,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): "extraction", "ocr", "realtime", + "rerank", ] ] supported_endpoints: list[str] | None diff --git a/litellm/utils.py b/litellm/utils.py index b0826f84006..38494cd1d99 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7327,6 +7327,7 @@ def get_model_info_helper( output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None), output_cost_per_video_per_second=_model_info.get("output_cost_per_video_per_second", None), output_cost_per_image=_model_info.get("output_cost_per_image", None), + input_cost_per_pixel=_model_info.get("input_cost_per_pixel", None), output_cost_per_pixel=_model_info.get("output_cost_per_pixel", None), output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None), output_cost_per_image_token_batches=_model_info.get("output_cost_per_image_token_batches", None), @@ -7531,6 +7532,11 @@ def get_model_info( "supported_openai_params": ["temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty"] } """ + from litellm.litellm_core_utils.llm_cost_calc.pricing_context import retained_model_info + + retained: Final = retained_model_info(model, custom_llm_provider) + if retained is not None: + return retained # api_key is a per-caller credential, not part of the model identity, so it is # kept out of the cache key; explicit keys are resolved without the cache. if api_key is not None: diff --git a/schema.prisma b/schema.prisma index 179b741fe7c..c2f34e3b17e 100644 --- a/schema.prisma +++ b/schema.prisma @@ -2111,8 +2111,12 @@ model LiteLLM_AccountingEvent { occurred_at DateTime recorded_at DateTime @default(now()) payload Json + compilation_state String @default("pending") + compilation_error String? + compilation_retry_at DateTime? @@index([recorded_at]) + @@index([kind, compilation_state, recorded_at]) @@index([kind, source_event_id]) } diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index 15cd5413149..0dc27b09991 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -20,6 +20,7 @@ from psycopg import sql from pydantic import JsonValue from litellm.accounting import ( + CostCompiler, CostEvent, DeliveryState, DeliveryTarget, @@ -30,9 +31,11 @@ from litellm.accounting import ( OutboundUsageEvent, Outcome, PricingSnapshot, + Unpriced, ) from litellm.proxy.db import accounting_ledger from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger, UsageWrite +from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler, price_usage from litellm.types.utils import CallTypes from tests.integration._support.database import scratch_database from tests.integration._support.prisma_clients import connected_proxy_prisma @@ -248,6 +251,184 @@ async def test_event_and_complete_delivery_set_commit_together(writer: _Writer, assert set(deliveries) == {(target.value, "pending", 0) for target in _TARGETS} +@pytest.mark.asyncio +async def test_compilers_share_one_committed_charge_and_preserve_delivery_receipts( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event, context={"price": "0.125"}) + + def price(envelope: str) -> PricingSnapshot: + return _cost(amount=json.loads(envelope)["context"]["price"]).snapshot + + compiler: Final[CostCompiler] = LoggingCostCompiler(store, price, _TARGETS) + results: Final = await asyncio.gather(*(compiler.compile(event) for _ in range(8))) + assert all(result == results[0] for result in results) + assert isinstance(results[0], CostEvent) + assert results[0].snapshot.total_cost == Decimal("0.125") + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(1,)] + deliveries: Final = _rows( + ledger_database, + 'SELECT target::text, state::text FROM "LiteLLM_AccountingDelivery" WHERE event_id = %s', + (results[0].event_id,), + ) + assert set(deliveries) == {(target.value, "pending") for target in _TARGETS} + _complete_and_age(ledger_database) + before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') + + def unavailable_pricing(envelope: str) -> PricingSnapshot: + raise AssertionError("Committed cost replay must not invoke pricing") + + replay: Final = await LoggingCostCompiler(store, unavailable_pricing, _TARGETS).compile(event) + assert replay == results[0] + assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before + + +@pytest.mark.asyncio +async def test_compilation_rejects_changed_delivery_targets_without_resetting_receipts( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + original: Final = await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS[:1]).compile(event) + _complete_and_age(ledger_database) + before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') + with pytest.raises(AccountingConflictError, match="delivery set"): + await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event) + assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before + assert await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS[:1]).compile(event) == original + + +@pytest.mark.asyncio +async def test_compilation_preserves_unknown_cost_without_creating_a_zero_charge( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + unresolved: Final = Unpriced(usage_event_id=event.event_id, reason="unknown_usage") + compiler: Final = LoggingCostCompiler(store, lambda _: unresolved, _TARGETS) + assert await compiler.compile(event) == Unpriced(usage_event_id=event.event_id, reason="usage_not_committed") + await _insert(store, event) + assert await compiler.compile(event) == unresolved + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] + + +@pytest.mark.asyncio +async def test_compilation_rejects_an_unpriced_result_for_another_source(writer: _Writer, ledger_database: str) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + compiler: Final = LoggingCostCompiler( + store, lambda _: Unpriced(usage_event_id="other", reason="missing_price"), _TARGETS + ) + with pytest.raises(AccountingConflictError, match="Unpriced result"): + await compiler.compile(event) + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] + + +@pytest.mark.asyncio +async def test_compilation_rejects_a_cost_identity_collision(writer: _Writer, ledger_database: str) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + await _insert(store, _usage(f"cost:{event.event_id}", inbound=True)) + with pytest.raises(AccountingConflictError, match="identity_conflict"): + await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event) + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] + + +@pytest.mark.asyncio +async def test_compilation_rejects_changed_source_before_reusing_a_charge(writer: _Writer) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + compiler: Final = LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS) + assert isinstance(await compiler.compile(event), CostEvent) + changed: Final = event.model_copy(update={"outcome": Outcome.CANCELLED}) + with pytest.raises(AccountingConflictError, match="differs from the accepted usage"): + await compiler.compile(changed) + + +@pytest.mark.asyncio +async def test_compilation_failure_retains_retry_without_a_partial_charge( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + + def failed_pricing(envelope: str) -> PricingSnapshot: + raise RuntimeError("pricing temporarily unavailable") + + assert await LoggingCostCompiler(store, failed_pricing, _TARGETS).compile(event) == Unpriced( + usage_event_id=event.event_id, reason="calculation_retryable" + ) + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] + result: Final = await LoggingCostCompiler(store, lambda _: _cost(amount="0").snapshot, _TARGETS).compile(event) + assert isinstance(result, CostEvent) + assert result.charges[0].value == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["usage", "cost"]) +async def test_compilation_does_not_interpret_unknown_envelope_versions(writer: _Writer, kind: str) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await store.insert_event(event, _TARGETS, envelope=_envelope(event), schema_version=2 if kind == "usage" else 1) + if kind == "cost": + cost: Final = _cost() + await store.insert_event(cost, _TARGETS, envelope=_envelope(cost), schema_version=2) + result: Final = await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event) + assert result == Unpriced(usage_event_id=event.event_id, reason=f"unsupported_{kind}_schema") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["before_commit", "after_commit", "cancellation"]) +async def test_compiler_recovery_preserves_atomic_charge_and_delivery_acceptance( + writer: _Writer, ledger_database: str, failure: str +) -> None: + client, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + + @asynccontextmanager + async def interrupted() -> AsyncGenerator[Prisma]: + async with client.writer_db.tx() as tx: + yield tx + if failure == "before_commit": + raise RuntimeError("interrupted cost commit") + if failure == "cancellation": + raise asyncio.CancelledError + raise RuntimeError("lost cost acknowledgement") + + compiler: Final = LoggingCostCompiler(AccountingLedger(interrupted), lambda _: _cost().snapshot, _TARGETS) + error: Final = asyncio.CancelledError if failure == "cancellation" else RuntimeError + with pytest.raises(error): + await compiler.compile(event) + count: Final = int(failure == "after_commit") + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(count,)] + assert _rows( + ledger_database, + 'SELECT count(*) FROM "LiteLLM_AccountingDelivery" WHERE event_id = %s', + (f"cost:{event.event_id}",), + ) == [(count * len(_TARGETS),)] + result: Final = await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event) + assert isinstance(result, CostEvent) + assert result.snapshot == _cost().snapshot + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(1,)] + + @pytest.mark.asyncio async def test_concurrent_duplicates_preserve_completed_deliveries(writer: _Writer, ledger_database: str) -> None: _, store = writer @@ -608,3 +789,345 @@ async def test_invalid_jsonb_submission_cannot_reject_another_producers_commit( assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("valid",)] assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)] assert (meter.pending_bytes, meter.pending_records) == (0, 0) + + +@pytest.mark.asyncio +async def test_compilation_persists_invalid_pricing_reason_without_charging( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + event: Final = _usage() + assert isinstance(event, OutboundUsageEvent) + await _insert(store, event) + accepted: Final = _rows( + ledger_database, + 'SELECT payload, payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = %s', + (event.event_id,), + ) + + def malformed(envelope: str) -> PricingSnapshot: + raise ValueError("invalid private input must not appear in durable diagnostics") + + result: Final = await LoggingCostCompiler(store, malformed, _TARGETS).compile(event) + assert result == Unpriced(usage_event_id=event.event_id, reason="invalid_pricing_inputs") + assert _rows( + ledger_database, + 'SELECT compilation_state, compilation_error FROM "LiteLLM_AccountingEvent" WHERE event_id = %s', + (event.event_id,), + ) == [("unpriced", "invalid_pricing_inputs")] + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)] + assert ( + _rows( + ledger_database, + 'SELECT payload, payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = %s', + (event.event_id,), + ) + == accepted + ) + async with connected_proxy_prisma(ledger_database) as restarted: + assert await AccountingLedger.for_client(restarted).pending_compilation() == () + + +@pytest.mark.asyncio +async def test_pending_compilation_only_returns_unresolved_billable_work(writer: _Writer, ledger_database: str) -> None: + _, store = writer + events: Final = tuple(_usage(name) for name in ("compiled", "excluded", "unknown", "pending")) + assert all(isinstance(event, OutboundUsageEvent) for event in events) + for event in events: + await _insert(store, event) + await _insert(store, _usage("inbound", inbound=True)) + compiled, excluded, unknown, pending = events + assert isinstance(compiled, OutboundUsageEvent) + assert isinstance(excluded, OutboundUsageEvent) + assert isinstance(unknown, OutboundUsageEvent) + assert isinstance(pending, OutboundUsageEvent) + assert isinstance( + await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(compiled), CostEvent + ) + await LoggingCostCompiler( + store, lambda _: Unpriced(usage_event_id=excluded.event_id, reason="not_charge_owner"), _TARGETS + ).compile(excluded) + await LoggingCostCompiler( + store, lambda _: Unpriced(usage_event_id=unknown.event_id, reason="usage_unknown"), _TARGETS + ).compile(unknown) + assert await store.pending_compilation(limit=1) == (pending,) + assert _rows( + ledger_database, + "SELECT event_id, compilation_state FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'outbound_usage' ORDER BY event_id", + ) == [("compiled", "compiled"), ("excluded", "excluded"), ("pending", "pending"), ("unknown", "unpriced")] + + +@pytest.mark.asyncio +async def test_restarted_compiler_recovers_transient_failures_without_revisiting_terminal_work( + writer: _Writer, ledger_database: str +) -> None: + _, store = writer + delayed: Final = _usage("delayed") + healthy: Final = _usage("healthy") + assert isinstance(delayed, OutboundUsageEvent) + assert isinstance(healthy, OutboundUsageEvent) + await _insert(store, delayed) + await _insert(store, healthy) + + def transient(envelope: str) -> PricingSnapshot: + if OutboundUsageEvent.model_validate(json.loads(envelope)["event"]).event_id == delayed.event_id: + raise RuntimeError("temporary pricing dependency failure") + return _cost().snapshot + + await LoggingCostCompiler(store, transient, _TARGETS).compile_pending() + assert await store.pending_compilation() == () + assert _rows( + ledger_database, + "SELECT compilation_state, compilation_error FROM \"LiteLLM_AccountingEvent\" WHERE event_id = 'delayed'", + ) == [("pending", "calculation_retryable")] + assert _rows(ledger_database, "SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [ + (healthy.event_id,) + ] + async with connected_proxy_prisma(ledger_database) as restarted: + recovered: Final = AccountingLedger.for_client(restarted) + await restarted.db.execute_raw( + 'UPDATE "LiteLLM_AccountingEvent" SET compilation_retry_at = NULL WHERE event_id = $1', delayed.event_id + ) + await LoggingCostCompiler(recovered, lambda _: _cost().snapshot, _TARGETS).compile_pending() + assert await recovered.pending_compilation() == () + await LoggingCostCompiler(recovered, transient, _TARGETS).compile_pending() + assert _rows( + ledger_database, + "SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost' ORDER BY source_event_id", + ) == [(delayed.event_id,), (healthy.event_id,)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_ack", (False, True)) +@pytest.mark.parametrize("provider_header", (False, True)) +async def test_proxy_capture_and_background_compiler_converge_on_final_header_charge( + writer: _Writer, ledger_database: str, monkeypatch: pytest.MonkeyPatch, sync_ack: bool, provider_header: bool +) -> None: + import litellm + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.spend_tracking.cost_finalization import finalize_response_cost + from litellm.proxy.spend_tracking.usage_capture import UsageBatchWriter, UsageCaptureRuntime, capture_proxy_response + from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + + _, store = writer + entered: Final = asyncio.Event() + release: Final = asyncio.Event() + batches: Final = UsageBatchWriter(store.insert_usage_batch, store.load_usage_batch) + + async def delayed_write(writes: tuple[UsageWrite, ...]) -> None: + entered.set() + await release.wait() + await batches.write(writes) + + compiler: Final = LoggingCostCompiler(store) + runtime: Final = UsageCaptureRuntime() + runtime.start(delayed_write, UsageCaptureSettings(batch_size=2), load=store.load_usage_batch, compiler=compiler) + monkeypatch.setattr(proxy_server, "usage_capture", runtime) + monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": sync_ack}) + logger: Final = Logging( + model="capture-priced-model", + messages=[], + stream=False, + call_type="acompletion", + start_time=_AT, + litellm_call_id="gateway-request", + function_id="gateway-attempt", + ) + logger.update_environment_variables( + model="capture-priced-model", custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + if not provider_header: + logger.set_request_pricing( + "capture-priced-model", "openai", {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02} + ) + response: Final = ModelResponse( + id="gateway-provider", model="capture-priced-model", usage=Usage(prompt_tokens=10, completion_tokens=4) + ) + if provider_header: + from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params + + set_response_cost_in_hidden_params(response, 0.18) + try: + finalized: Final = await finalize_response_cost(logger, response, None, _AT, guardrail_cost=0.03) + assert finalized is not None + headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=UserAPIKeyAuth(), response_cost=finalized.amount, litellm_logging_obj=logger + ) + capture: Final = asyncio.create_task(capture_proxy_response(logger, response)) + await asyncio.wait_for(entered.wait(), timeout=5) + if sync_ack: + assert not capture.done() + else: + await asyncio.wait_for(asyncio.shield(capture), timeout=5) + assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(0,)] + release.set() + await asyncio.wait_for(capture, timeout=10) + assert runtime.meter is not None + await runtime.meter.flush() + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [ + (1 if sync_ack else 0,) + ] + await asyncio.gather(*(compiler.compile_pending() for _ in range(3))) + costs: Final = _rows( + ledger_database, "SELECT payload::text FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'" + ) + assert len(costs) == 1 + cost_envelope: Final = json.loads(str(costs[0][0])) + committed: Final = CostEvent.model_validate_json(json.dumps(cost_envelope["event"])) + assert committed.snapshot.total_cost == Decimal(headers["x-litellm-response-cost"]) + assert float(committed.snapshot.total_cost) == pytest.approx(0.21) + monkeypatch.setattr(litellm, "model_cost", {"openai/capture-priced-model": {"input_cost_per_token": 99.0}}) + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.99}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 999.0}}) + replayed: Final = price_usage(json.dumps(cost_envelope["pricing_inputs"])) + assert replayed == committed.snapshot + assert litellm.model_cost["openai/capture-priced-model"]["input_cost_per_token"] == 99.0 + await capture_proxy_response(logger, response) + await compiler.compile_pending() + assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(1,)] + finally: + release.set() + await runtime.stop(5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("conflicting_result", (False, True)) +async def test_failing_compilation_page_does_not_starve_later_usage( + writer: _Writer, ledger_database: str, conflicting_result: bool +) -> None: + _, store = writer + broken: Final = _usage("broken-price") + healthy: Final = _usage("healthy-price") + assert isinstance(broken, OutboundUsageEvent) + assert isinstance(healthy, OutboundUsageEvent) + await _insert(store, broken) + await _insert(store, healthy) + + def price(envelope: str) -> PricingSnapshot | Unpriced: + if OutboundUsageEvent.model_validate(json.loads(envelope)["event"]).event_id == broken.event_id: + if conflicting_result: + return Unpriced(usage_event_id="different-usage", reason="usage_unknown") + raise RuntimeError("temporary calculator failure") + return _cost().snapshot + + await LoggingCostCompiler(store, price, _TARGETS).compile_pending(limit=1) + assert await store.pending_compilation(limit=1) == (healthy,) + _rows( + ledger_database, + 'UPDATE "LiteLLM_AccountingEvent" SET compilation_retry_at = CURRENT_TIMESTAMP WHERE event_id = %s', + (broken.event_id,), + ) + await LoggingCostCompiler(store, price, _TARGETS).compile_pending(limit=1) + assert _rows(ledger_database, "SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [ + (healthy.event_id,) + ] + assert isinstance(await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(broken), CostEvent) + assert await store.pending_compilation() == () + + +@pytest.mark.asyncio +@pytest.mark.parametrize("summary_first", (False, True)) +async def test_batch_callbacks_commit_only_row_charges_across_restart( + writer: _Writer, ledger_database: str, monkeypatch: pytest.MonkeyPatch, summary_first: bool +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import ( + UsageBatchWriter, + capture_batch_callback, + capture_callback, + capture_logging_usage, + ) + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + _, store = writer + rates: Final = { + "input_cost_per_token_batches": 0.003, + "output_cost_per_token_batches": 0.004, + "litellm_provider": "openai", + "mode": "chat", + } + monkeypatch.setitem(litellm.model_cost, "capture-model", rates) + usage: Final = {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} + response: Final = { + "id": "managed-job", + "status": "completed", + "usage": { + "batch_usage_by_model": {"capture-model": usage}, + "batch_usage_by_request": [{"model": "capture-model", "usage": usage} for _ in range(2)], + }, + } + logger: Final = _logger("aretrieve_batch") + batches: Final = UsageBatchWriter(store.insert_usage_batch, store.load_usage_batch) + meter: Final = DurableUsageMeter(batches.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) + compiler: Final = LoggingCostCompiler(store) + capture_logging_usage(meter, logger, response, _AT, _AT, freeze_only=True) + kwargs: Final = {**logger.model_call_details, "litellm_logging_obj": logger} + + async def summary() -> None: + capture_callback( + meter, + kwargs, + { + "id": "provider-job", + "status": "completed", + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + }, + _AT, + _AT, + ) + await meter.flush() + await compiler.compile_pending() + + try: + if summary_first: + await summary() + await capture_batch_callback(meter, store.load_usage_batch, kwargs, response, _AT, _AT) + await compiler.compile_pending() + if not summary_first: + await summary() + before: Final = _rows( + ledger_database, 'SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent" ORDER BY event_id' + ) + costs: Final = _rows( + ledger_database, "SELECT payload::text FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'" + ) + deliveries: Final = _rows( + ledger_database, + 'SELECT event_id, target, state FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target', + ) + assert len(costs) == 2 + snapshots: Final = tuple( + CostEvent.model_validate_json(json.dumps(json.loads(str(row[0]))["event"])).snapshot for row in costs + ) + assert sum(float(snapshot.total_cost) for snapshot in snapshots) == pytest.approx(2 * (10 * 0.003 + 5 * 0.004)) + monkeypatch.setitem(litellm.model_cost, "capture-model", {**rates, "input_cost_per_token_batches": 99.0}) + async with connected_proxy_prisma(ledger_database) as restarted: + ledger: Final = AccountingLedger.for_client(restarted) + from litellm.proxy.openai_files_endpoints.common_utils import encode_file_id_with_model + + for job_id in ("managed-job", encode_file_id_with_model("managed-job", "capture-model", "batch")): + await capture_batch_callback( + meter, ledger.load_usage_batch, kwargs, {**response, "id": job_id}, _AT, _AT + ) + await asyncio.gather(*(LoggingCostCompiler(ledger).compile_pending() for _ in range(2))) + assert ( + _rows(ledger_database, 'SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent" ORDER BY event_id') + == before + ) + assert ( + _rows( + ledger_database, + 'SELECT event_id, target, state FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target', + ) + == deliveries + ) + for row, snapshot in zip(costs, snapshots, strict=True): + assert price_usage(json.dumps(json.loads(str(row[0]))["pricing_inputs"])) == snapshot + finally: + await meter.stop(1) diff --git a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py index 5e40c2ebb60..4c253a10116 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py @@ -549,7 +549,7 @@ async def test_completion_sagemaker( assert route.called assert response.choices[0].message.content == "SageMaker reply" - assert litellm.completion_cost(completion_response=response) == pytest.approx(rate) + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate) @pytest.mark.asyncio diff --git a/tests/unit/proxy/spend_tracking/test_cost_compiler.py b/tests/unit/proxy/spend_tracking/test_cost_compiler.py new file mode 100644 index 00000000000..077a7b8b379 --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_cost_compiler.py @@ -0,0 +1,1769 @@ +import json +from datetime import datetime, timezone +from decimal import Decimal +from typing import Final + +import pytest + +from litellm.accounting import Dimension, DimensionName, OutboundUsageEvent, Outcome, PricingSnapshot, Unpriced +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.proxy.spend_tracking.cost_compiler import price_usage +from litellm.types.utils import CallTypes + +_AT: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + + +def _envelope() -> dict[str, object]: + event: Final = OutboundUsageEvent( + event_id="attempt", + attempt_id="attempt", + request_id="request", + retry_of_attempt_id=None, + call_type=CallTypes.acompletion, + started_at=_AT, + ended_at=_AT, + outcome=Outcome.SUCCEEDED, + dimensions=( + Dimension(name=DimensionName.MODEL, value="openai/retained-model"), + Dimension(name=DimensionName.PROVIDER, value="openai"), + ), + metrics=None, + ) + return { + "event": event.model_dump(mode="json"), + "charge_eligible": True, + "usage": {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14}, + "pricing": { + "model_map_information": { + "model_map_key": "openai/retained-model", + "model_map_value": { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "litellm_provider": "openai", + "mode": "chat", + }, + }, + "adjustments": {"discount": 0.1, "margin": {"fixed_amount": 0.008}}, + }, + } + + +def test_compiler_prices_retained_rates_without_registering_a_model() -> None: + import litellm + + result: Final = price_usage(json.dumps(_envelope())) + assert isinstance(result, PricingSnapshot) + assert result.total_cost == Decimal(str((10 * 0.01 + 4 * 0.02) * 0.9 + 0.008)) + assert "openai/retained-model" not in litellm.model_cost + + +@pytest.mark.parametrize( + ("field", "value", "reason"), + [ + ("usage", None, "usage_unknown"), + ("charge_eligible", False, "not_charge_owner"), + ("pricing", {}, "pricing_inputs_missing"), + ("finalized_cost", {"response_cost": 999}, "finalized_cost_mismatch"), + ], +) +def test_compiler_does_not_invent_a_charge(field: str, value: object, reason: str) -> None: + result: Final = price_usage(json.dumps({**_envelope(), field: value})) + assert result == Unpriced(usage_event_id="attempt", reason=reason) + + +def _retained_envelope( + *, + rates: dict[str, object] | None = None, + usage: dict[str, object] | None = None, + logging: dict[str, object] | None = None, + response_inputs: dict[str, object] | None = None, + deployment: dict[str, object] | None = None, + request: dict[str, object] | None = None, +) -> dict[str, object]: + from pydantic import TypeAdapter + + original: Final = _envelope() + pricing: Final = TypeAdapter(dict[str, object]).validate_python(original["pricing"]) + mapped: Final = TypeAdapter(dict[str, object]).validate_python(pricing["model_map_information"]) + base_rates: Final = TypeAdapter(dict[str, object]).validate_python(mapped["model_map_value"]) + return { + **original, + "usage": usage if usage is not None else original["usage"], + "pricing": { + **pricing, + "model_map_information": {**mapped, "model_map_value": {**base_rates, **(rates or {})}}, + "logging": logging or {}, + "response_inputs": response_inputs or {}, + "deployment": deployment or {}, + "request": request or {}, + "adjustments": {"discount": None, "margin": None}, + }, + } + + +def test_retained_pricing_ignores_later_global_price_and_adjustment_changes(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + + captured: Final = json.dumps(_envelope()) + original: Final = price_usage(captured) + monkeypatch.setattr(litellm, "model_cost", {"openai/retained-model": {"input_cost_per_token": 99.0}}) + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.99}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 999.0}}) + recomputed: Final = price_usage(captured) + assert isinstance(original, PricingSnapshot) + assert recomputed == original + assert litellm.model_cost["openai/retained-model"]["input_cost_per_token"] == 99.0 + + +def test_repricing_recomputes_retained_usage_instead_of_returning_a_reported_total() -> None: + from pydantic import TypeAdapter + + captured: Final = _retained_envelope() + pricing: Final = TypeAdapter(dict[str, object]).validate_python(captured["pricing"]) + original: Final = price_usage(json.dumps({**captured, "pricing": {**pricing, "reported_cost": 999.0}})) + changed: Final = price_usage(json.dumps({**captured, "usage": {"prompt_tokens": 20, "completion_tokens": 8}})) + assert isinstance(original, PricingSnapshot) + assert isinstance(changed, PricingSnapshot) + assert float(original.total_cost) == pytest.approx(0.18) + assert float(changed.total_cost) == pytest.approx(float(original.total_cost) * 2) + + +def test_retained_token_pricing_preserves_cache_reasoning_and_service_tier() -> None: + captured: Final = _retained_envelope( + rates={ + "input_cost_per_token_priority": 0.03, + "output_cost_per_token_priority": 0.05, + "cache_read_input_token_cost_priority": 0.002, + "output_cost_per_reasoning_token_priority": 0.07, + }, + usage={ + "prompt_tokens": 100, + "completion_tokens": 20, + "prompt_tokens_details": {"cached_tokens": 40}, + "completion_tokens_details": {"reasoning_tokens": 5}, + }, + response_inputs={"service_tier": "priority"}, + ) + result: Final = price_usage(json.dumps(captured)) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(60 * 0.03 + 40 * 0.002 + 15 * 0.05 + 5 * 0.07) + + +@pytest.mark.parametrize("stamped_cost", [None, 0.9]) +def test_upstream_reported_price_takes_precedence_without_custom_pricing(stamped_cost: float | None) -> None: + result: Final = price_usage( + json.dumps( + _retained_envelope( + logging={"provider_reported_cost": 0.037}, + usage={"prompt_tokens": 10, "completion_tokens": 4, "cost": stamped_cost}, + ) + ) + ) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.037) + + +def test_per_request_pricing_overrides_upstream_reported_price() -> None: + result: Final = price_usage( + json.dumps( + _retained_envelope( + logging={ + "provider_reported_cost": 0.037, + "custom_pricing": True, + "custom_cost_per_token": {"input_cost_per_token": 0.04, "output_cost_per_token": 0.06}, + } + ) + ) + ) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(10 * 0.04 + 4 * 0.06) + + +def test_retained_regional_pricing_preserves_data_residency_uplift() -> None: + result: Final = price_usage( + json.dumps( + _retained_envelope( + rates={"regional_processing_uplift_multiplier_eu": 1.1}, + deployment={"data_residency": "eu"}, + ) + ) + ) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx((10 * 0.01 + 4 * 0.02) * 1.1) + + +def test_retained_request_additional_cost_is_applied_once() -> None: + result: Final = price_usage(json.dumps(_retained_envelope(request={"additional_response_cost": 0.3}))) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(10 * 0.01 + 4 * 0.02 + 0.3) + + +def _capture_logger(call_type: str = "acompletion", *, stream: bool = False) -> Logging: + from tests.unit.proxy.spend_tracking.test_usage_capture import _logger + + logger: Final = _logger(call_type, stream=stream) + logger.set_request_pricing("capture-model", "openai", {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}) + return logger + + +@pytest.mark.asyncio +async def test_real_capture_preserves_synchronous_finalization_and_guardrail_parity() -> None: + from litellm.proxy.spend_tracking.cost_finalization import finalize_response_cost + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger() + response: Final = ModelResponse( + id="finalized", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4, total_tokens=14) + ) + finalized: Final = await finalize_response_cost(logger, response, None, _AT, guardrail_cost=0.03) + assert finalized is not None + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(finalized.amount) + assert float(result.guardrail_cost or 0) == pytest.approx(0.03) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_stream_capture_only_prices_final_usage() -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, ModelResponseStream, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger(stream=True) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, ModelResponseStream(id="chunk"), _AT, _AT) + assert meter.pending_records == 0 + capture_logging_usage( + meter, + logger, + ModelResponse(id="terminal", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)), + _AT, + _AT, + ) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.18) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_internal_capture_excludes_usage_owned_by_parent() -> None: + from litellm._internal_context import internal_sub_call + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + with internal_sub_call("parent"): + capture_logging_usage( + meter, + _capture_logger(), + ModelResponse(id="child", model="capture-model", usage=Usage(prompt_tokens=10)), + _AT, + _AT, + ) + await meter.close() + assert price_usage(database.rows[-1].envelope) == Unpriced( + usage_event_id=database.rows[-1].event.event_id, reason="not_charge_owner" + ) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("already_calculated", (False, True)) +async def test_real_mcp_capture_prices_retained_tool_fee_without_token_usage(already_calculated: bool) -> None: + from mcp.types import CallToolResult, TextContent + + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger(CallTypes.call_mcp_tool.value) + logger.model_call_details["mcp_tool_call_metadata"] = { + "name": "lookup", + "namespaced_tool_name": "server.lookup", + "mcp_server_cost_info": {"default_cost_per_query": 0.2, "tool_name_to_cost_per_query": {"lookup": 0.3}}, + } + if already_calculated: + logger.model_call_details["response_cost"] = 0.3 + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, CallToolResult(content=[TextContent(type="text", text="ok")]), _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.3) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_agent_capture_prices_retained_per_query_fee() -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.agents import LiteLLMSendMessageResponse + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger(CallTypes.asend_message.value) + logger.update_environment_variables( + model="capture-model", custom_llm_provider="a2a", litellm_params={"cost_per_query": 0.4}, optional_params={} + ) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, LiteLLMSendMessageResponse(id="agent"), _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.4) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reported_cost", (0.0, 0.123)) +@pytest.mark.parametrize("stamped", (False, True)) +@pytest.mark.parametrize("adjusted", (False, True)) +async def test_real_capture_prices_upstream_cost_without_a_model_map( + monkeypatch: pytest.MonkeyPatch, reported_cost: float, stamped: bool, adjusted: bool +) -> None: + import litellm + from litellm.litellm_core_utils.core_helpers import stamp_litellm_bill_on_usage + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.05} if adjusted else {}) + monkeypatch.setattr( + litellm, "cost_margin_config", {"global": {"percentage": 0.1, "fixed_amount": 0.001}} if adjusted else {} + ) + expected: Final = reported_cost * 0.95 * 1.1 + 0.001 if adjusted else reported_cost + logger: Final = _logger() + + response: Final = ModelResponse( + id="upstream", model="capture-model", usage=Usage(prompt_tokens=10, cost=reported_cost) + ) + if stamped: + stamp_litellm_bill_on_usage(response, response.usage, reported_cost + 0.9) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(expected) + assert result.upstream_reported_cost == Decimal(str(reported_cost)) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancelled", (False, True)) +async def test_real_failure_capture_preserves_incurred_charge(cancelled: bool) -> None: + from fastapi import HTTPException + + from litellm.litellm_core_utils.litellm_logging import UsageObservation + from litellm.proxy.spend_tracking.usage_capture import capture_callback, capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger() + usage: Final = Usage(prompt_tokens=4, completion_tokens=0, total_tokens=4) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage( + meter, logger, ModelResponse(model="capture-model", usage=usage), _AT, _AT, freeze_only=True + ) + capture_callback( + meter, + { + **logger.model_call_details, + "litellm_logging_obj": logger, + "combined_usage_object": usage, + "_accounting_usage_observed": UsageObservation.KNOWN, + "log_event_type": "failed_api_call", + }, + HTTPException(499, "Client disconnected") if cancelled else RuntimeError("provider failed after billing"), + _AT, + _AT, + ) + await meter.close() + outbound: Final = database.rows[-1] + assert outbound.event.outcome is (Outcome.CANCELLED if cancelled else Outcome.FAILED) + result: Final = price_usage(outbound.envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.04) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_captured_price_replay_uses_original_off_peak_clock(monkeypatch: pytest.MonkeyPatch) -> None: + from datetime import timedelta + + import litellm + from litellm._internal_context import pinned_billing_time + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger() + monkeypatch.setitem( + litellm.model_cost, + "capture-model", + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "off_peak_pricing": { + "hours_utc": "00:00-01:00", + "input_cost_per_token": 0.005, + "output_cost_per_token": 0.01, + }, + }, + ) + response: Final = ModelResponse(model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + with pinned_billing_time(_AT): + source_cost: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + with pinned_billing_time(_AT + timedelta(hours=3)): + replay: Final = price_usage(database.rows[-1].envelope) + assert source_cost == pytest.approx(0.09) + assert isinstance(replay, PricingSnapshot) + assert float(replay.total_cost) == pytest.approx(source_cost) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_realtime_capture_preserves_audio_and_text_token_prices() -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger(CallTypes.arealtime.value) + logger.set_request_pricing( + "capture-model", + "openai", + { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "input_cost_per_audio_token": 0.03, + "output_cost_per_audio_token": 0.04, + }, + ) + events: Final = [ + { + "type": "response.done", + "response": { + "id": "turn", + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_token_details": {"text_tokens": 4, "audio_tokens": 6, "cached_tokens": 0}, + "output_token_details": {"text_tokens": 2, "audio_tokens": 3}, + }, + "output": "private transcript excluded from accounting", + }, + } + ] + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, events, _AT, _AT) + await meter.close() + captured: Final = database.rows[-1].envelope + assert "private transcript" not in captured + result: Final = price_usage(captured) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(4 * 0.01 + 6 * 0.03 + 2 * 0.02 + 3 * 0.04) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_realtime_capture_preserves_metered_audio_and_message_fees() -> None: + from litellm.constants import REALTIME_METERED_USAGE_KEY + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.xai import RealtimeMeteredUsage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger(CallTypes.arealtime.value) + logger.update_environment_variables( + model="capture-model", custom_llm_provider="xai", litellm_params={}, optional_params={} + ) + logger.set_request_pricing( + "capture-model", + "xai", + { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "input_cost_per_second": 0.02, + "input_cost_per_text_message": 0.1, + }, + ) + logger.model_call_details[REALTIME_METERED_USAGE_KEY] = RealtimeMeteredUsage(True, 1.5, 2.5, 3) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage( + meter, + logger, + [ + { + "type": "response.done", + "response": { + "id": "metered", + "usage": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + }, + } + ], + _AT, + _AT, + ) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx((1.5 + 2.5) * 0.02 + 3 * 0.1) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_real_batch_rows_keep_per_request_tiers_across_repeated_terminal_retrieval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.batches.batch_utils import batch_cost_and_usage + from litellm.proxy.spend_tracking.usage_capture import ( + UsageBatchWriter, + capture_batch_callback, + capture_logging_usage, + ) + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + rates: Final = { + "input_cost_per_token_batches": 0.001, + "input_cost_per_token_above_200k_tokens_batches": 0.002, + "output_cost_per_token_batches": 0.0, + "litellm_provider": "openai", + "mode": "chat", + } + monkeypatch.setitem(litellm.model_cost, "capture-model", rates) + rows: Final = [ + { + "response": { + "status_code": 200, + "body": { + "model": "capture-model", + "usage": {"prompt_tokens": tokens, "completion_tokens": 0, "total_tokens": tokens}, + }, + } + } + for tokens in (10, 200001) + ] + billed: Final = batch_cost_and_usage(rows, "openai") + response: Final = {"id": "batch", "status": "completed", "usage": billed.usage} + database: Final = StoredUsage() + writer: Final = UsageBatchWriter(database.insert, database.load) + meter: Final = DurableUsageMeter(writer.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) + logger: Final = _logger("aretrieve_batch") + logger.model_call_details["response_cost"] = billed.cost + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await capture_batch_callback(meter, database.load, logger.model_call_details, response, _AT, _AT) + await meter.flush() + first: Final = tuple(database.rows) + await capture_batch_callback(meter, database.load, logger.model_call_details, response, _AT, _AT) + await meter.close() + assert tuple(database.rows) == first + assert price_usage(database.rows[0].envelope) == Unpriced( + usage_event_id=database.rows[0].event.event_id, reason="not_charge_owner" + ) + costs: Final = tuple(price_usage(row.envelope) for row in database.rows[1:]) + assert all(isinstance(cost, PricingSnapshot) for cost in costs), costs + assert sum(float(cost.total_cost) for cost in costs if isinstance(cost, PricingSnapshot)) == pytest.approx( + 10 * 0.001 + 200001 * 0.002 + ) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("transcription", (False, True)) +async def test_realtime_capture_replays_sanitized_lifecycle_and_transcription_charges(transcription: bool) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger(CallTypes.arealtime.value) + logger.set_request_pricing( + "capture-model", + "openai", + { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "input_cost_per_second": 0.02, + }, + ) + events: Final = ( + [ + { + "type": "transcription_session.created", + "session": {"input_audio_transcription": {"model": "capture-model"}}, + }, + { + "type": "conversation.item.input_audio_transcription.completed", + "item_id": "audio", + "transcript": "private transcript", + "usage": {"type": "duration", "seconds": 2.5}, + }, + ] + if transcription + else [ + {"type": "session.started", "session": {"model": "capture-model"}}, + {"type": "session.closed", "usage": {"seconds": 2.5}, "transcript": "private transcript"}, + ] + ) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, events, _AT, _AT) + await meter.close() + captured: Final = database.rows[-1].envelope + assert "private transcript" not in captured + result: Final = price_usage(captured) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(2.5 * 0.02) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("file_search", (False, True)) +async def test_capture_replays_builtin_search_charge_without_retaining_results(file_search: bool) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.openai import ResponsesAPIResponse + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger("aresponses") + logger.set_request_pricing( + "capture-model", + "openai", + { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "search_context_cost_per_query": {"search_context_size_medium": 0.1}, + "file_search_cost_per_1k_calls": 0.025, + }, + ) + logger.standard_built_in_tools_params = ( + {"file_search": {"type": "file_search", "vector_store_ids": ["store"]}} + if file_search + else {"web_search_options": {"search_context_size": "medium"}} + ) + response: Final = ResponsesAPIResponse.model_validate( + { + "id": "search", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "capture-model", + "output": [ + {"type": "file_search_call", "id": "search", "queries": ["private query"], "status": "completed"} + ] + if file_search + else [ + { + "type": "web_search_call", + "id": "search", + "status": "completed", + "action": {"type": "search", "query": "private query"}, + } + ], + "usage": {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14}, + } + ) + source_cost: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert source_cost is not None and source_cost > 0.18 + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + captured: Final = database.rows[-1].envelope + assert "private query" not in captured + result: Final = price_usage(captured) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(source_cost) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_capture_replays_multi_round_aggregate_with_one_fixed_request_margin( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.responses.mcp.round_billing import billed_for_every_round, summed_cost_breakdown + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.1}}) + first_logger: Final = _capture_logger() + final_logger: Final = _capture_logger() + first: Final = ModelResponse(id="first", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)) + final: Final = ModelResponse(id="final", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)) + first._hidden_params["response_cost"] = first_logger.response_cost_calculator(first, ignore_stamped_cost=True) + final._hidden_params["response_cost"] = final_logger.response_cost_calculator(final, ignore_stamped_cost=True) + aggregate: Final = billed_for_every_round(first, first_logger.cost_breakdown, final, final_logger.cost_breakdown) + breakdown: Final = summed_cost_breakdown(first, first_logger.cost_breakdown, final, final_logger.cost_breakdown) + final_logger.cost_breakdown = breakdown + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, final_logger, aggregate, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(2 * (10 * 0.01 + 4 * 0.02) + 0.1) + assert float(result.total_cost) == pytest.approx(aggregate._hidden_params["response_cost"]) + finally: + await meter.stop(1) + + +@pytest.mark.parametrize("reported", (-0.1, float("inf"), float("-inf"), float("nan"), True, False)) +@pytest.mark.parametrize("has_rates", (False, True)) +def test_invalid_upstream_price_cannot_become_an_authoritative_charge(reported: float, has_rates: bool) -> None: + from pydantic import TypeAdapter + + envelope: Final = _retained_envelope(logging={"provider_reported_cost": reported}) + pricing: Final = TypeAdapter(dict[str, object]).validate_python(envelope["pricing"]) + if not has_rates: + pricing["model_map_information"] = {} + envelope["pricing"] = pricing + result: Final = price_usage(json.dumps(envelope)) + if has_rates: + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.18) + assert result.upstream_reported_cost is None + else: + assert result == Unpriced(usage_event_id="attempt", reason="pricing_inputs_missing") + + +def test_repricing_rejects_unretained_allowance_allocation() -> None: + envelope: Final = _retained_envelope() + envelope["finalized_cost"] = {"allowance_discount": 0.1} + assert price_usage(json.dumps(envelope)) == Unpriced( + usage_event_id="attempt", reason="allowance_allocation_missing" + ) + + +@pytest.mark.asyncio +async def test_retained_pricing_is_task_local_and_resets_on_cancellation() -> None: + import asyncio + + from litellm.litellm_core_utils.llm_cost_calc.pricing_context import ( + RetainedPricing, + pricing_discounts, + retained_model_info, + retained_models, + retained_pricing, + ) + + outer: Final = RetainedPricing("{}", '{"openai":0.1}', "{}") + inner: Final = RetainedPricing("{}", '{"openai":0.9}', "{}") + ready: Final = asyncio.Event() + hold: Final = asyncio.Event() + restored: Final = asyncio.Event() + + async def child() -> None: + try: + with retained_pricing(inner): + assert pricing_discounts()["openai"] == 0.9 + with pytest.raises(ValueError, match="no retained model"): + retained_model_info("gpt-4o-mini", "openai") + ready.set() + await hold.wait() + finally: + assert pricing_discounts()["openai"] == 0.1 + restored.set() + + assert retained_models() is None + with retained_pricing(outer): + task: Final = asyncio.create_task(child()) + try: + await asyncio.wait_for(ready.wait(), timeout=5) + assert pricing_discounts()["openai"] == 0.1 + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert restored.is_set() + assert pricing_discounts()["openai"] == 0.1 + assert retained_models() is None + assert retained_model_info("gpt-4o-mini", "openai") is None + + +def test_incomplete_retained_rates_remain_unpriced() -> None: + result: Final = price_usage( + json.dumps(_retained_envelope(rates={"input_cost_per_token": None, "output_cost_per_token": None})) + ) + assert result == Unpriced(usage_event_id="attempt", reason="pricing_inputs_missing") + + +def test_explicit_zero_retained_token_rates_are_known_free() -> None: + result: Final = price_usage( + json.dumps(_retained_envelope(rates={"input_cost_per_token": 0.0, "output_cost_per_token": 0.0})) + ) + assert isinstance(result, PricingSnapshot) + assert result.total_cost == Decimal("0") + assert result.pricing_source == "cost_map" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("completion_tokens", [1, 4]) +async def test_stream_replay_applies_adjustments_once_to_legacy_stamped_usage( + monkeypatch: pytest.MonkeyPatch, completion_tokens: int +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.05}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"percentage": 0.1, "fixed_amount": 0.001}}) + retained: Final = _capture_logger().request_pricing + assert retained is not None + monkeypatch.setitem(litellm.model_cost, "openai/capture-model", retained.models()["capture-model"]) + monkeypatch.setitem(litellm.model_cost, "capture-model", retained.models()["capture-model"]) + logger: Final = _logger(stream=True) + response: Final = ModelResponse( + id="terminal", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=completion_tokens) + ) + expected: Final = _logger(stream=True).response_cost_calculator(response, ignore_stamped_cost=True) + assert expected is not None + from litellm.litellm_core_utils.core_helpers import stamp_litellm_bill_on_usage + + stamp_litellm_bill_on_usage(response, response.usage, expected) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + priced: Final = price_usage(database.rows[-1].envelope) + assert isinstance(priced, PricingSnapshot) + assert float(priced.total_cost) == pytest.approx(expected) + assert float(priced.original_cost or 0) == pytest.approx(0.1 + completion_tokens * 0.02) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("amount", [0.0, 0.123]) +@pytest.mark.parametrize("mapped", [None, "mapped-cost-model", "openai/mapped-cost-model"]) +@pytest.mark.parametrize("provider", ["openai", "litellm_proxy"]) +async def test_capture_retains_provider_header_price_without_catalog( + amount: float, mapped: str | None, provider: str, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + response: Final = ModelResponse( + model="capture-model", + usage=Usage(prompt_tokens=10, completion_tokens=4), + ) + from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params + + set_response_cost_in_hidden_params(response, amount) + import litellm + + logger: Final = _logger() + logger.update_environment_variables( + model="capture-model", custom_llm_provider=provider, litellm_params={}, optional_params={} + ) + if mapped: + monkeypatch.setitem( + litellm.model_cost, + "mapped-cost-model", + { + "mode": "chat", + "litellm_provider": "openai", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + }, + ) + logger.model_call_details["litellm_params"]["metadata"] = {"model_info": {"base_model": mapped}} + expected: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + expected_breakdown: Final = logger.cost_breakdown + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + if mapped: + monkeypatch.setitem(litellm.model_cost, "mapped-cost-model", {"input_cost_per_token": 99.0}) + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert result.total_cost == Decimal(str(amount)) + assert float(result.total_cost) == expected + assert expected_breakdown is not None + assert float(result.input_cost or 0) == pytest.approx(expected_breakdown["input_cost"]) + assert float(result.output_cost or 0) == pytest.approx(expected_breakdown["output_cost"]) + assert (result.input_cost or Decimal(0)) + (result.output_cost or Decimal(0)) == result.total_cost + assert result.pricing_source == "upstream_header" + finally: + await meter.stop(1) + + +@pytest.mark.parametrize("metadata", [{"input_cost_per_token": None}, {"service_tier": "default"}]) +def test_catalog_provenance_ignores_metadata_without_price_overrides(metadata: dict[str, object]) -> None: + result: Final = price_usage(json.dumps(_retained_envelope(deployment=metadata, request=metadata))) + assert isinstance(result, PricingSnapshot) + assert result.pricing_source == "cost_map" + + +@pytest.mark.parametrize("call_type", ["get_responses", "aget_responses"]) +def test_terminal_response_replay_prices_usage_without_changing_event_identity(call_type: str) -> None: + from pydantic import TypeAdapter + + original: Final = _retained_envelope() + event: Final = TypeAdapter(dict[str, object]).validate_python(original["event"]) + result: Final = price_usage(json.dumps({**original, "event": {**event, "call_type": call_type}})) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(10 * 0.01 + 4 * 0.02) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("freeze_pricing", [False, True]) +async def test_batch_terminal_capture_refreshes_row_prices_after_creation( + monkeypatch: pytest.MonkeyPatch, freeze_pricing: bool +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_batch_callback, capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + rates: Final = { + "mode": "chat", + "litellm_provider": "openai", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "input_cost_per_token_batches": 0.003, + "output_cost_per_token_batches": 0.004, + } + monkeypatch.setitem(litellm.model_cost, "capture-model", rates) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.1}}) + monkeypatch.setitem(litellm.model_cost, "second-model", {**rates, "input_cost_per_token_batches": 0.005}) + logger: Final = _logger("aretrieve_batch") + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + usage: Final = {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14} + response: Final = { + "id": "batch", + "status": "completed", + "usage": { + **usage, + "batch_usage_by_model": {"capture-model": usage, "second-model": usage}, + "batch_usage_by_request": [ + {"model": "capture-model", "usage": usage}, + {"model": "capture-model", "usage": usage}, + {"model": "second-model", "usage": usage}, + ], + }, + } + try: + capture_logging_usage(meter, logger, {"id": "batch", "status": "in_progress"}, _AT, _AT, freeze_only=True) + if freeze_pricing: + capture_logging_usage(meter, logger, response, _AT, _AT) + await capture_batch_callback( + meter, database.load, {**logger.model_call_details, "litellm_logging_obj": logger}, response, _AT, _AT + ) + first_rows: Final = database.rows + monkeypatch.setitem(litellm.model_cost, "capture-model", {**rates, "input_cost_per_token_batches": 99.0}) + capture_logging_usage(meter, logger, response, _AT, _AT) + await capture_batch_callback(meter, database.load, logger.model_call_details, response, _AT, _AT) + await meter.close() + assert database.rows == first_rows + results: Final = tuple(price_usage(row.envelope) for row in database.rows) + priced: Final = tuple(result for result in results if isinstance(result, PricingSnapshot)) + assert len(priced) == 3 + assert sorted(float(price.total_cost) for price in priced) == pytest.approx([0.046, 0.046, 0.066]) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["get_responses", "aget_responses"]) +async def test_background_capture_compiles_terminal_usage_once(call_type: str) -> None: + from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger(call_type) + logger.update_environment_variables( + model="capture-model", + custom_llm_provider="openai", + optional_params={}, + litellm_params={"metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN}}, + ) + response: Final = ResponsesAPIResponse.model_validate( + { + "id": "background-job", + "object": "response", + "created_at": int(_AT.timestamp()), + "status": "completed", + "background": True, + "model": "capture-model", + "output": [], + "usage": {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14}, + } + ) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + assert len(database.rows) == 1 + assert database.rows[0].event.call_type.value == call_type + assert database.rows[0].event.event_id == "background-job_background_cost" + result: Final = price_usage(database.rows[0].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(10 * 0.01 + 4 * 0.02) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["call_mcp_tool", "asend_message"]) +async def test_failed_fixed_call_without_usage_does_not_invent_a_free_charge(call_type: str) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_callback + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger(call_type) + logger.model_call_details["response_cost"] = 0.0 + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_callback( + meter, + {**logger.model_call_details, "litellm_logging_obj": logger, "log_event_type": "failed_api_call"}, + RuntimeError("Request rejected before dispatch"), + _AT, + _AT, + ) + await meter.close() + outbound: Final = database.rows[-1] + assert outbound.event.outcome is Outcome.FAILED + assert price_usage(outbound.envelope) == Unpriced( + usage_event_id=outbound.event.event_id, reason="usage_unknown" + ) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("billable", [False, True]) +async def test_capture_preserves_completed_guardrail_fee_without_header_finalization( + stream: bool, billable: bool +) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger(stream=stream) + logger.model_call_details["litellm_params"]["metadata"] = { + "standard_logging_guardrail_information": [ + {"guardrail_name": "late-fee", "guardrail_cost": 0.03, "guardrail_cost_in_spend": billable} + ] + } + response: Final = ModelResponse( + id="guarded", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4, total_tokens=14) + ) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert result.guardrail_cost == Decimal("0.03" if billable else "0") + assert float(result.total_cost) == pytest.approx(0.18 + (0.03 if billable else 0)) + finally: + await meter.stop(1) + + +@pytest.mark.parametrize("mode", [None, "chat"]) +def test_header_split_retains_token_rates_when_catalog_mode_is_unspecified(mode: str | None) -> None: + result: Final = price_usage( + json.dumps(_retained_envelope(rates={"mode": mode}, logging={"provider_reported_cost": 0.18})) + ) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(0.18) + assert float(result.input_cost or 0) == pytest.approx(0.1) + assert float(result.output_cost or 0) == pytest.approx(0.08) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("logging_first", (False, True)) +async def test_batch_logging_and_rows_have_one_charge_owner( + monkeypatch: pytest.MonkeyPatch, logging_first: bool +) -> None: + import litellm + from litellm.batches.batch_utils import batch_cost_and_usage + from litellm.proxy.spend_tracking.usage_capture import ( + UsageBatchWriter, + capture_batch_callback, + capture_callback, + capture_logging_usage, + ) + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import LiteLLMBatch + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + rates: Final = { + "input_cost_per_token_batches": 0.003, + "output_cost_per_token_batches": 0.004, + "input_cost_per_token": 0.006, + "output_cost_per_token": 0.008, + "litellm_provider": "openai", + "mode": "chat", + } + monkeypatch.setitem(litellm.model_cost, "capture-model", rates) + billed: Final = batch_cost_and_usage( + [ + { + "response": { + "status_code": 200, + "body": { + "model": "capture-model", + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + } + } + for _ in range(2) + ], + "openai", + ) + response: Final = LiteLLMBatch( + id="owner-job", + object="batch", + endpoint="/v1/chat/completions", + input_file_id="file", + completion_window="24h", + status="completed", + created_at=1, + completed_at=2, + usage=billed.usage, + ) + database: Final = StoredUsage() + writer: Final = UsageBatchWriter(database.insert, database.load) + meter: Final = DurableUsageMeter(writer.write, UsageCaptureSettings(), wait_for_batch=_manual_flush) + logger: Final = _logger("aretrieve_batch") + capture_logging_usage(meter, logger, response, _AT, _AT, freeze_only=True) + try: + for _ in range(2): + if logging_first: + capture_callback( + meter, + {**logger.model_call_details, "litellm_logging_obj": logger}, + { + **response.model_dump(), + "id": "provider-job", + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + }, + _AT, + _AT, + ) + capture_logging_usage(meter, logger, response, _AT, _AT) + await capture_batch_callback( + meter, database.load, {**logger.model_call_details, "litellm_logging_obj": logger}, response, _AT, _AT + ) + if not logging_first: + capture_callback( + meter, + {**logger.model_call_details, "litellm_logging_obj": logger}, + { + **response.model_dump(), + "id": "provider-job", + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}, + }, + _AT, + _AT, + ) + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.flush() + costs: Final = tuple( + price_usage(row.envelope) for row in database.rows if isinstance(row.event, OutboundUsageEvent) + ) + priced: Final = tuple(cost for cost in costs if isinstance(cost, PricingSnapshot)) + assert len(priced) == 2, costs + assert sum(float(cost.total_cost) for cost in priced) == pytest.approx(2 * (10 * 0.003 + 5 * 0.004)) + finally: + await meter.stop(1) + + +@pytest.mark.parametrize("characters", (0, 17)) +def test_speech_replay_prices_retained_characters(characters: int) -> None: + original: Final = _retained_envelope( + rates={"input_cost_per_character": 0.003, "mode": "audio_speech"}, logging={"prompt_characters": characters} + ) + event: Final = OutboundUsageEvent.model_validate(original["event"]).model_copy( + update={"call_type": CallTypes.aspeech} + ) + result: Final = price_usage(json.dumps({**original, "event": event.model_dump(mode="json")})) + assert isinstance(result, PricingSnapshot), result + assert float(result.total_cost) == pytest.approx(characters * 0.003) + + +def test_batch_ocr_replay_prices_pages_and_annotations() -> None: + original: Final = _retained_envelope( + rates={"ocr_cost_per_page_batches": 0.003, "annotation_cost_per_page_batches": 0.004, "mode": "ocr"}, + usage={"pages_processed": 3, "pages_processed_annotation": 2}, + ) + event: Final = OutboundUsageEvent.model_validate(original["event"]).model_copy( + update={"call_type": CallTypes.aretrieve_batch} + ) + result: Final = price_usage( + json.dumps( + { + **original, + "event": {**event.model_dump(mode="json"), "event_id": "batch:row:0"}, + "parent_request_id": "batch", + } + ) + ) + assert isinstance(result, PricingSnapshot), result + assert float(result.total_cost) == pytest.approx(3 * 0.003 + 2 * 0.004) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reported_characters", (None, 5)) +async def test_speech_capture_replays_billed_quantities_without_input_text(reported_characters: int | None) -> None: + from httpx import Response + + from litellm.litellm_core_utils.hidden_params import set_hidden_param + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.openai import HttpxBinaryResponseContent + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _logger("aspeech") + logger.set_request_pricing("capture-model", "openai", {"input_cost_per_character": 0.003, "mode": "audio_speech"}) + logger.model_call_details["input"] = "privatespeechinput" + response: Final = HttpxBinaryResponseContent(Response(200, content=b"audio")) + if reported_characters is not None: + set_hidden_param(response, "billed_characters", reported_characters) + expected: Final = (reported_characters if reported_characters is not None else len("privatespeechinput")) * 0.003 + assert logger.response_cost_calculator(response) == pytest.approx(expected) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.flush() + envelope: Final = next(row.envelope for row in database.rows if isinstance(row.event, OutboundUsageEvent)) + assert "privatespeechinput" not in envelope + priced: Final = price_usage(envelope) + assert isinstance(priced, PricingSnapshot), priced + assert float(priced.total_cost) == pytest.approx(expected) + finally: + await meter.stop(1) + + +@pytest.mark.parametrize("characters", (None, -1, True)) +def test_character_pricing_does_not_invent_zero_for_missing_or_invalid_quantity(characters: object) -> None: + original: Final = _retained_envelope( + rates={"input_cost_per_character": 0.003}, logging={"prompt_characters": characters} + ) + event: Final = OutboundUsageEvent.model_validate(original["event"]).model_copy( + update={"call_type": CallTypes.aspeech} + ) + assert price_usage(json.dumps({**original, "event": event.model_dump(mode="json")})) == Unpriced( + usage_event_id=event.event_id, reason="usage_unknown" + ) + + +@pytest.mark.parametrize("measured_tokens", (False, True)) +def test_rerank_replay_preserves_billable_search_units(measured_tokens: bool) -> None: + from pydantic import TypeAdapter + + original: Final = _retained_envelope( + rates={"mode": "rerank", "input_cost_per_query": 0.03}, + response_inputs={"billed_units": {"search_units": 3, "total_tokens": 14}}, + ) + event: Final = OutboundUsageEvent.model_validate(original["event"]).model_copy( + update={ + "call_type": CallTypes.arerank, + "dimensions": ( + Dimension(name=DimensionName.MODEL, value="cohere/retained-reranker"), + Dimension(name=DimensionName.PROVIDER, value="cohere"), + ), + } + ) + pricing: Final = TypeAdapter(dict[str, object]).validate_python(original["pricing"]) + result: Final = price_usage( + json.dumps( + { + **original, + "usage": original["usage"] if measured_tokens else None, + "event": event.model_dump(mode="json"), + "pricing": pricing, + } + ) + ) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(3 * 0.03) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("served_tier", ("default", "priority")) +async def test_responses_replay_uses_served_tier(served_tier: str) -> None: + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.openai import ResponsesAPIResponse + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + logger: Final = _capture_logger("aresponses") + logger.set_request_pricing( + "capture-model", + "openai", + { + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + "input_cost_per_token_priority": 0.03, + "output_cost_per_token_priority": 0.04, + }, + ) + logger.update_environment_variables( + model="capture-model", + custom_llm_provider="openai", + litellm_params={}, + optional_params={"service_tier": "priority" if served_tier == "default" else "default"}, + ) + response: Final = ResponsesAPIResponse.model_validate( + { + "id": "tier", + "created_at": 1, + "model": "capture-model", + "output": [], + "service_tier": served_tier, + "usage": {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14}, + } + ) + source_cost: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert source_cost == pytest.approx(0.18 if served_tier == "default" else 0.46) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(source_cost) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("token_pricing", "automatic_size"), ((False, False), (False, True), (True, False))) +@pytest.mark.parametrize("custom_primary", (False, True)) +async def test_responses_image_tool_replays_retained_prices_without_image_content( + monkeypatch: pytest.MonkeyPatch, token_pricing: bool, custom_primary: bool, automatic_size: bool +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.llms.openai import ResponsesAPIResponse + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setitem( + litellm.model_cost, + "capture-model", + {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, + ) + monkeypatch.setitem( + litellm.model_cost, + "retained-image", + { + "litellm_provider": "openai", + "mode": "image_generation", + "input_cost_per_token": 0.03, + "output_cost_per_image_token": 0.04, + }, + ) + image_price_key: Final = f"{'high' if automatic_size else 'low'}/1024-x-1024/retained-image" + monkeypatch.setitem( + litellm.model_cost, + image_price_key, + {"litellm_provider": "openai", "mode": "image_generation", "output_cost_per_image": 0.25}, + ) + logger: Final = _logger("aresponses") + if custom_primary: + logger.set_request_pricing( + "capture-model", "openai", {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02} + ) + response: Final = ResponsesAPIResponse.model_validate( + { + "id": "image-tool", + "created_at": 1, + "model": "capture-model", + "tools": [{"type": "image_generation", "model": "retained-image"}], + "output": [ + { + "id": "image", + "type": "image_generation_call", + "status": "completed", + "quality": "auto" if automatic_size else "low", + "size": "auto" if automatic_size else "1024x1024", + "result": "private-image-content", + } + ], + "usage": {"input_tokens": 10, "output_tokens": 4, "total_tokens": 14}, + "tool_usage": { + "image_gen": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"text_tokens": 10, "image_tokens": 0}, + "output_tokens_details": {"text_tokens": 0, "image_tokens": 5}, + } + } + if token_pricing + else None, + } + ) + source_cost: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert source_cost == pytest.approx(0.18 + (0.5 if token_pricing else 0.25)) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + captured: Final = database.rows[-1].envelope + assert "private-image-content" not in captured + monkeypatch.delitem(litellm.model_cost, "retained-image") + monkeypatch.delitem(litellm.model_cost, image_price_key) + result: Final = price_usage(captured) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(source_cost) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("spilled_over", (False, True)) +async def test_capture_replays_effective_ptu_spillover_prices( + monkeypatch: pytest.MonkeyPatch, spilled_over: bool +) -> None: + import litellm + from litellm.litellm_core_utils.hidden_params import set_hidden_param + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", "true") + monkeypatch.setitem( + litellm.model_cost, + "azure/capture-model", + {"litellm_provider": "azure", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, + ) + logger: Final = _logger() + logger.update_environment_variables( + model="capture-model", + custom_llm_provider="azure", + optional_params={}, + litellm_params={ + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "metadata": { + "model_info": { + "team_id": "team", + "ptu_count": 10, + "cost_per_ptu_per_hour": 1.0, + "ptu_effective_from": "2026-01-01", + } + }, + }, + ) + logger.set_request_pricing("capture-model", "azure", {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}) + response: Final = ModelResponse( + id="spill", model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4, total_tokens=14) + ) + set_hidden_param( + response, "additional_headers", {"llm_provider-x-ms-is-spilled-over": "true"} if spilled_over else {} + ) + source_cost: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert source_cost == pytest.approx(0.18 if spilled_over else 0) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + result: Final = price_usage(database.rows[-1].envelope) + assert isinstance(result, PricingSnapshot) + assert float(result.total_cost) == pytest.approx(source_cost) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_live_adjustment_reload_is_captured_and_replay_remains_frozen(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setattr(litellm, "cost_discount_config", {}) + monkeypatch.setattr(litellm, "cost_margin_config", {}) + logger: Final = _capture_logger() + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.1}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.02}}) + response: Final = ModelResponse(model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + amount: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert amount == pytest.approx(0.18 * 0.9 + 0.02) + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.9}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 99.0}}) + replay: Final = price_usage(database.rows[-1].envelope) + assert isinstance(replay, PricingSnapshot) + assert float(replay.total_cost) == pytest.approx(amount) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("pricing", "width", "expected"), + ( + ("tokens", 2048, 0.2), + ("images", 2048, 0.5), + ("pixels", 2048, 0.4), + ("pixels", 2049, 0.6), + ("tiered", 2048, 1.5), + ("tiered-request", 2048, 1.5), + ("azure-tiered", 2048, 1.5), + ), +) +async def test_image_capture_replays_the_existing_image_calculator( + monkeypatch: pytest.MonkeyPatch, pricing: str, width: int, expected: float +) -> None: + import litellm + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ImageResponse + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage, _logger + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.1}) + monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 0.2}}) + model: Final = { + "tokens": "gpt-image-capture", + "images": "dall-e-capture", + "pixels": "fal-ai/flux/dev", + "tiered": "dall-e-capture", + "tiered-request": "dall-e-capture", + "azure-tiered": "dall-e-3", + }[pricing] + provider: Final = "fal_ai" if pricing == "pixels" else "azure" if pricing == "azure-tiered" else "openai" + catalog_provider: Final = provider + catalog_key: Final = f"{provider}/{model}" if pricing == "pixels" else model + monkeypatch.setitem( + litellm.model_cost, + catalog_key, + { + "litellm_provider": catalog_provider, + "mode": "image_generation", + **{ + "tokens": {"input_cost_per_token": 0.01, "output_cost_per_image_token": 0.02}, + "images": {"output_cost_per_image": 0.25}, + "tiered": {"output_cost_per_image": 0.25}, + "tiered-request": {"output_cost_per_image": 0.25}, + "azure-tiered": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, + "pixels": {"output_cost_per_image": 0.1, "output_cost_per_pixel": 0.1 / (1024 * 1024)}, + }[pricing], + }, + ) + tier_key: Final = f"{provider}/hd/1024-x-1024/{model}" if pricing == "azure-tiered" else f"hd/1024-x-1024/{model}" + if "tiered" in pricing: + monkeypatch.setitem( + litellm.model_cost, + tier_key, + { + "litellm_provider": catalog_provider, + "mode": "image_generation", + **( + {"input_cost_per_pixel": 0.75 / (1024 * 1024)} + if pricing == "azure-tiered" + else {"output_cost_per_image": 0.75} + ), + }, + ) + logger: Final = _logger("aimage_generation") + logger.model = model + logger.update_environment_variables( + model=model, + custom_llm_provider=provider, + litellm_params={}, + optional_params={"size": "1024*1024", "quality": "hd"} if pricing == "tiered-request" else {}, + ) + response: Final = ImageResponse.model_validate( + { + "created": 1, + "model": model, + "data": [ + { + "b64_json": "private-image-content", + "provider_specific_fields": { + "width": width, + "height": 1024, + "secret": "private-image-content", + }, + } + ] + * 2, + "size": None if pricing == "tiered-request" else "1024x1024", + "quality": None if pricing == "tiered-request" else "hd" if "tiered" in pricing else "standard", + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "input_tokens_details": {"text_tokens": 10, "image_tokens": 0}, + } + if pricing == "tokens" + else None, + } + ) + source: Final = logger.response_cost_calculator(response, ignore_stamped_cost=True) + assert source == pytest.approx(expected) + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + capture_logging_usage(meter, logger, response, _AT, _AT) + await meter.close() + envelope: Final = database.rows[-1].envelope + assert "private-image-content" not in envelope + monkeypatch.delitem(litellm.model_cost, catalog_key) + monkeypatch.delitem(litellm.model_cost, tier_key, raising=False) + replay: Final = price_usage(envelope) + assert isinstance(replay, PricingSnapshot) + assert float(replay.total_cost) == pytest.approx(source) + finally: + await meter.stop(1) + + +@pytest.mark.asyncio +async def test_delayed_attempt_capture_keeps_its_prices_after_fallback_reset(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.main import _set_request_pricing + from litellm.proxy.spend_tracking.usage_capture import capture_logging_usage + from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings + from litellm.types.utils import ModelResponse, Usage + from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage + from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush + + catalog: Final = { + "openai/capture-model": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + } + monkeypatch.setattr(litellm, "model_cost", catalog) + monkeypatch.setattr(litellm, "cost_discount_config", {}) + monkeypatch.setattr(litellm, "cost_margin_config", {}) + logger: Final = _capture_logger() + failed_attempt: Final = logger.attempt_scoped_copy() + _set_request_pricing(logger, "capture-model", "openai", {}, None) + response: Final = ModelResponse(model="capture-model", usage=Usage(prompt_tokens=10, completion_tokens=4)) + for attempt, expected in ((logger, 0.018), (failed_attempt, 0.18)): + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + assert attempt.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(expected) + capture_logging_usage(meter, attempt, response, _AT, _AT) + await meter.close() + with monkeypatch.context() as changed_catalog: + changed_catalog.setattr(litellm, "model_cost", {}) + replay: Final = price_usage(database.rows[-1].envelope) + assert isinstance(replay, PricingSnapshot) + assert float(replay.total_cost) == pytest.approx(expected) + finally: + await meter.stop(1) diff --git a/tests/unit/proxy/spend_tracking/test_usage_capture.py b/tests/unit/proxy/spend_tracking/test_usage_capture.py index 32710e594ee..a1327e0a177 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_capture.py +++ b/tests/unit/proxy/spend_tracking/test_usage_capture.py @@ -1,13 +1,13 @@ import asyncio import json -from collections.abc import Awaitable, Mapping +from collections.abc import Mapping from datetime import datetime, timezone from decimal import Decimal from typing import Final import pytest -from litellm.accounting import Metric, MetricName, Outcome +from litellm.accounting import Metric, MetricName, OutboundUsageEvent, Outcome, Unpriced from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureError, UsageCaptureSettings @@ -334,7 +334,7 @@ async def test_competing_writers_reuse_only_terminal_pricing_snapshots(terminal: @pytest.mark.asyncio -async def test_batch_poll_has_distinct_identity_and_only_terminal_usage_is_chargeable() -> None: +async def test_batch_poll_and_terminal_summary_are_not_charge_owners() -> None: from litellm.proxy.spend_tracking.usage_capture import capture_callback database: Final = StoredUsage() @@ -352,8 +352,8 @@ async def test_batch_poll_has_distinct_identity_and_only_terminal_usage_is_charg await meter.close() assert pending.writes[-1].event.event_id == "poll:status" assert json.loads(pending.writes[-1].envelope)["charge_eligible"] is False - assert completed.writes[-1].event.event_id == "batch_batch_cost" - assert json.loads(completed.writes[-1].envelope)["charge_eligible"] is True + assert completed.writes[-1].event.event_id == "batch_batch_cost:summary" + assert json.loads(completed.writes[-1].envelope)["charge_eligible"] is False assert len(completed.writes) == 1 @@ -441,13 +441,13 @@ async def test_supported_sync_ack_waits_for_commit(call_type: str, monkeypatch: monkeypatch.setattr(proxy_server, "usage_capture", runtime) monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True}) deadlines: Final = asyncio.Queue[float | None]() - original_wait: Final = asyncio.wait_for + original_timeout: Final = asyncio.timeout - async def wait_with_deadline(awaitable: Awaitable[object], timeout: float | None) -> object: - deadlines.put_nowait(timeout) - return await original_wait(awaitable, timeout) + def timeout_with_deadline(delay: float | None) -> asyncio.Timeout: + deadlines.put_nowait(delay) + return original_timeout(delay) - monkeypatch.setattr(asyncio, "wait_for", wait_with_deadline) + monkeypatch.setattr(asyncio, "timeout", timeout_with_deadline) response: Final = asyncio.create_task( capture_proxy_response( _logger(call_type), @@ -1021,9 +1021,8 @@ async def test_batch_capture_provenance_tracks_observations_not_envelope_presenc envelope: Final = json.loads(database.rows[-1].envelope) assert envelope["usage_source"] == ("unknown" if measured is None else "measured") assert envelope["usage"]["batch_usage_by_model"] == usage["batch_usage_by_model"] - assert database.rows[-1].event.metrics == ( - None if measured is None else (Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(measured["total_tokens"])),) - ) + assert database.rows[-1].event.metrics is None + assert envelope["charge_eligible"] is False @pytest.mark.asyncio @@ -1075,17 +1074,14 @@ async def test_response_ack_timeout_preserves_success_and_later_commits(monkeypa runtime.start(database.write, UsageCaptureSettings(batch_size=2)) monkeypatch.setattr(proxy_server, "usage_capture", runtime) monkeypatch.setattr(proxy_server, "prisma_client", None) - monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True}) + monkeypatch.setattr( + proxy_server, "general_settings", {"accounting_sync_ack": True, "accounting_sync_ack_timeout_seconds": 0.01} + ) logger: Final = _logger() - async def expired_deadline(future: object, timeout: float) -> None: - raise TimeoutError("controlled response deadline") - try: - with monkeypatch.context() as clock: - clock.setattr(asyncio, "wait_for", expired_deadline) - with pytest.raises(HTTPException) as error: - await capture_proxy_response(logger, ModelResponse(id="provider", usage=Usage(prompt_tokens=3))) + with pytest.raises(HTTPException) as error: + await capture_proxy_response(logger, ModelResponse(id="provider", usage=Usage(prompt_tokens=3))) assert error.value.status_code == 503 await database.entered.wait() captured: Final = logger.model_call_details["_accounting_capture"] @@ -1710,3 +1706,80 @@ async def test_batch_replay_reserves_the_accepted_pricing_snapshot_size() -> Non assert runtime.meter is not None and runtime.meter.pending_bytes == 0 finally: await runtime.stop(1) + + +class DelayedCostCompiler: + def __init__(self) -> None: + self.entered: Final = asyncio.Event() + self.cancelled: Final = asyncio.Event() + self.release: Final = asyncio.Event() + + async def compile(self, event: OutboundUsageEvent) -> Unpriced: + self.entered.set() + try: + await self.release.wait() + except asyncio.CancelledError: + self.cancelled.set() + raise + return Unpriced(usage_event_id=event.event_id, reason="usage_unknown") + + +@pytest.mark.asyncio +async def test_sync_ack_preserves_committed_usage_ack_when_compilation_times_out( + monkeypatch: pytest.MonkeyPatch, +) -> None: + + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response + from litellm.types.utils import ModelResponse, Usage + + database: Final = CommitBarrier() + database.release.set() + compiler: Final = DelayedCostCompiler() + runtime: Final = UsageCaptureRuntime() + runtime.start(database.write, UsageCaptureSettings(batch_size=2), compiler=compiler) + monkeypatch.setattr(proxy_server, "usage_capture", runtime) + monkeypatch.setattr( + proxy_server, "general_settings", {"accounting_sync_ack": True, "accounting_sync_ack_timeout_seconds": 0.1} + ) + try: + await asyncio.wait_for( + capture_proxy_response(_logger(), ModelResponse(id="provider", usage=Usage(prompt_tokens=3))), 2 + ) + assert compiler.entered.is_set() + assert compiler.cancelled.is_set() + assert database.committed == ("caller:inbound", "provider") + finally: + compiler.release.set() + await runtime.stop(1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["create_batch", "acreate_batch"]) +async def test_batch_creation_is_not_a_charge_owner(call_type: str) -> None: + from litellm.proxy.spend_tracking.cost_compiler import price_usage + from litellm.proxy.spend_tracking.usage_capture import capture_callback + from litellm.accounting import Unpriced + + database: Final = StoredUsage() + meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush) + try: + captured: Final = capture_callback( + meter, + { + "call_type": call_type, + "litellm_call_id": "create", + "model": "capture-model", + "litellm_params": {}, + }, + {"id": "batch", "status": "validating"}, + NOW, + NOW, + ) + await meter.close() + outbound: Final = captured.writes[-1] + assert price_usage(outbound.envelope) == Unpriced( + usage_event_id=outbound.event.event_id, reason="not_charge_owner" + ) + finally: + await meter.stop(1) diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index cca138e0bb5..bb4035e8725 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -11108,3 +11108,50 @@ def test_upstream_cost_header_total_on_an_unmapped_alias_is_split_by_the_served_ input_share: Final = (10 * 0.001) / (10 * 0.001 + 20 * 0.002) assert breakdown["input_cost"] == pytest.approx(0.25 * input_share) assert breakdown["output_cost"] == pytest.approx(0.25 * (1 - input_share)) + + +@pytest.mark.parametrize( + ("model", "retained_key", "deployment_rate"), + ( + ("video-model", "openai/video-model", None), + ("openai/video-model", "video-model", None), + ("openai/video-model", "openai/video-model", 0.7), + ), +) +def test_video_pricing_uses_retained_catalog_and_deployment_precedence( + monkeypatch: pytest.MonkeyPatch, model: str, retained_key: str, deployment_rate: float | None +) -> None: + from pydantic import TypeAdapter + + from litellm.cost_calculator import default_video_cost_calculator + from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing + from litellm.types.utils import ModelInfo + + info: Final = TypeAdapter(ModelInfo).validate_python( + { + "key": retained_key, + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": None, + "output_cost_per_token": None, + "litellm_provider": "openai", + "mode": "image_generation", + "supported_openai_params": None, + "output_cost_per_video_per_second": 0.2, + } + ) + retained: Final = RetainedPricing(json.dumps({retained_key: info}), "{}", "{}") + changed: Final = {**info, "output_cost_per_video_per_second": 9.0} + monkeypatch.setattr(litellm, "model_cost", {"video-model": changed, "openai/video-model": changed}) + deployment: Final = ( + TypeAdapter(ModelInfo).validate_python({**info, "output_cost_per_video_per_second": deployment_rate}) + if deployment_rate is not None + else None + ) + with retained_pricing(retained): + price: Final = default_video_cost_calculator( + model, duration_seconds=4, custom_llm_provider="openai", model_info=deployment + ) + assert price == pytest.approx(4 * (deployment_rate if deployment_rate is not None else 0.2)) + assert litellm.model_cost["video-model"]["output_cost_per_video_per_second"] == 9.0 diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 35fbd28eae7..d971f0d7014 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -1,13 +1,4 @@ -""" -Test that register_model() in completion() and embedding() passes all -custom pricing fields from kwargs and model_info, not just the base -input/output costs. - -Previously, only input_cost_per_token, output_cost_per_token, and -litellm_provider were forwarded. Fields like cache_read_input_token_cost, -mode, and supports_prompt_caching were dropped, causing incorrect cost -calculations for DB-sourced models with prompt caching pricing. -""" +"""Verify explicit model registration and isolated request pricing preserve effective rates.""" import copy import itertools @@ -17,9 +8,9 @@ from typing import Final, NoReturn import pytest - import litellm from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.main import _build_custom_pricing_entry from litellm.utils import _invalidate_model_cost_lowercase_map @@ -646,17 +637,15 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): """ shared_key = "openai/text-embedding-3-small" deployment_id = "lit3991-wildcard-embed-zero" - snapshot = _snapshot_model_cost_entries( - [shared_key, "text-embedding-3-small", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost > 0 + logger: Final = _request_logger(shared_key, "embedding") try: - litellm.embedding( + response = litellm.embedding( model=shared_key, + litellm_logging_obj=logger, input=["hello"], api_key="fake-key", input_cost_per_token=0.0, @@ -666,12 +655,10 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): mock_response=[0.1, 0.2], ) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ), "wildcard deployment's zero pricing leaked into the shared model_cost key" - assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0 - assert litellm.model_cost[deployment_id]["output_cost_per_token"] == 0.0 + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, ( + "wildcard deployment's zero pricing leaked into the shared model_cost key" + ) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == 0.0 sibling_response = litellm.embedding( model=shared_key, @@ -679,9 +666,7 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): api_key="fake-key", mock_response=[0.1, 0.2], ) - sibling_cost = litellm.completion_cost( - completion_response=sibling_response, call_type="embedding" - ) + sibling_cost = litellm.completion_cost(completion_response=sibling_response, call_type="embedding") assert sibling_cost == pytest.approx(10 * builtin_input_cost) finally: _restore_model_cost_entries(snapshot) @@ -695,17 +680,15 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): shared_key = "openai/text-embedding-3-small" deployment_id = "lit3991-wildcard-embed-custom" override_input_cost = 5e-05 - snapshot = _snapshot_model_cost_entries( - [shared_key, "text-embedding-3-small", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost != override_input_cost + logger: Final = _request_logger(shared_key, "embedding") try: response = litellm.embedding( model=shared_key, + litellm_logging_obj=logger, input=["hello"], api_key="fake-key", input_cost_per_token=override_input_cost, @@ -715,19 +698,9 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): mock_response=[0.1, 0.2], ) - request_cost = litellm.completion_cost( - completion_response=response, - model=shared_key, - custom_llm_provider="openai", - call_type="embedding", - custom_pricing=True, - router_model_id=deployment_id, - ) + request_cost = logger.response_cost_calculator(response, ignore_stamped_cost=True) assert request_cost == pytest.approx(10 * override_input_cost) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ) + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost finally: _restore_model_cost_entries(snapshot) @@ -736,17 +709,15 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): """Same isolation as the embedding path, exercised through completion().""" shared_key = "openai/gpt-4o-mini" deployment_id = "lit3991-wildcard-chat-zero" - snapshot = _snapshot_model_cost_entries( - [shared_key, "gpt-4o-mini", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "gpt-4o-mini", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost > 0 + logger: Final = _request_logger(shared_key, "completion") try: - litellm.completion( + response = litellm.completion( model=shared_key, + litellm_logging_obj=logger, messages=[{"role": "user", "content": "hello"}], api_key="fake-key", input_cost_per_token=0.0, @@ -756,47 +727,43 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): mock_response="hello back", ) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ), "wildcard deployment's zero pricing leaked into the shared model_cost key" - assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0 + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, ( + "wildcard deployment's zero pricing leaked into the shared model_cost key" + ) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == 0.0 finally: _restore_model_cost_entries(snapshot) -def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): - """Direct SDK calls (no router deployment id in metadata) keep the legacy - behavior: custom pricing is registered under ``{provider}/{model}`` and the - request is costed with it. - """ - model_key = "openai/lit3991-direct-sdk-embed-model" - override_input_cost = 3e-05 - try: - response = litellm.embedding( - model=model_key, - input=["hello"], - api_key="fake-key", - input_cost_per_token=override_input_cost, - output_cost_per_token=override_input_cost * 2, - mock_response=[0.1, 0.2], - ) +def _request_logger(model: str, call_type: str) -> Logging: + from datetime import datetime, timezone - assert ( - litellm.model_cost[model_key]["input_cost_per_token"] - == override_input_cost - ) - cost = litellm.completion_cost( - completion_response=response, - model=model_key, - custom_llm_provider="openai", - call_type="embedding", - custom_pricing=True, - ) - assert cost == pytest.approx(10 * override_input_cost) - finally: - litellm.model_cost.pop(model_key, None) - _invalidate_model_cost_lowercase_map() + return Logging( + model=model, + messages=[], + stream=False, + call_type=call_type, + start_time=datetime.now(timezone.utc), + litellm_call_id="override", + function_id="override", + ) + + +def test_embedding_direct_sdk_custom_pricing_is_request_scoped() -> None: + model: Final = "openai/request-priced-embedding" + before: Final = _snapshot_model_cost_entries((model,)) + logger: Final = _request_logger(model, "embedding") + response: Final = litellm.embedding( + model=model, + input=["hello"], + api_key="fake-key", + input_cost_per_token=0.003, + output_cost_per_token=0.006, + mock_response=[0.1, 0.2], + litellm_logging_obj=logger, + ) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(10 * 0.003) + assert _snapshot_model_cost_entries((model,)) == before @pytest.mark.parametrize("route", ["", "converse/", "invoke/"]) @@ -1407,10 +1374,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing(): object stays untouched, and the caller's incoming nested dict is never inserted by reference into the merged result. """ - from litellm.utils import update_dictionary - from typing import cast + from litellm.utils import update_dictionary + existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"}) existing = {"off_peak_pricing": existing_nested} incoming_nested = cast( @@ -1609,22 +1576,24 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing(): del router -def test_completion_registers_cost_per_second_pricing(): - model_key: Final = "openai/test-cost-per-second-registration" - original_entries: Final = _snapshot_model_cost_entries([model_key]) +def test_completion_per_second_pricing_is_request_scoped() -> None: + from litellm.types.utils import ModelResponse, Usage - try: - litellm.completion( - model=model_key, - messages=[{"role": "user", "content": "hello"}], - api_key="fake-key", - cost_per_second=0.02, - mock_response="hello back", - ) - - assert litellm.model_cost[model_key]["cost_per_second"] == 0.02 - finally: - _restore_model_cost_entries(original_entries) + model: Final = "openai/request-priced-seconds" + logger: Final = _request_logger(model, "completion") + before: Final = _snapshot_model_cost_entries((model,)) + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + cost_per_second=0.02, + mock_response="hello back", + litellm_logging_obj=logger, + ) + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=0, completion_tokens=0)) + response._response_ms = 2000 + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(0.04) + assert _snapshot_model_cost_entries((model,)) == before def test_update_model_cost(): @@ -1676,31 +1645,346 @@ def test_register_model_scopes_builtin_match_to_the_given_provider(): _restore_model_cost_entries(model_cost_entries) -def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider(): - from litellm.main import _register_custom_pricing_for_request +def test_per_request_custom_pricing_preserves_a_colliding_deployment() -> None: + from litellm.main import _set_request_pricing + from litellm.types.utils import ModelResponse, Usage - deployment_id: Final = "baseten/zai-org/glm-5.2" - builtin_key: Final = "baseten/zai-org/GLM-5.2" - shared_key: Final = "openai/zai-org/GLM-5.2" - model_cost_entries: Final = _snapshot_model_cost_entries((deployment_id, builtin_key, shared_key)) - builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key]) - openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models) - try: - _register_custom_pricing_for_request( - model="zai-org/GLM-5.2", - custom_llm_provider="openai", - kwargs={ - "input_cost_per_token": 0.00000096, - "output_cost_per_token": 0.00000302, - "metadata": {"model_info": {"id": deployment_id}}, - }, - model_info={"mode": "chat"}, + deployment: Final = "baseten/zai-org/glm-5.2" + model: Final = "zai-org/GLM-5.2" + keys: Final = (deployment, "baseten/zai-org/GLM-5.2", f"openai/{model}") + before: Final = _snapshot_model_cost_entries(keys) + logger: Final = _request_logger(model, "completion") + logger.update_environment_variables( + model=model, user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai" + ) + _set_request_pricing( + logger, + model, + "openai", + {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "metadata": {"model_info": {"id": deployment}}}, + {"mode": "chat"}, + ) + response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5)) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(0.02) + assert _snapshot_model_cost_entries(keys) == before + + +@pytest.mark.parametrize("call_type", ("completion", "embedding")) +def test_concurrent_requests_keep_distinct_prices_and_leave_default_rates(call_type: str) -> None: + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + + model: Final = "openai/gpt-4o-mini" if call_type == "completion" else "openai/text-embedding-3-small" + before: Final = _snapshot_model_cost_entries((model, model.removeprefix("openai/"))) + ready: Final = Barrier(2) + + def execute(logger: Logging, rate: float) -> object: + if call_type == "completion": + return litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + input_cost_per_token=rate, + output_cost_per_token=rate, + mock_response="hello", + litellm_logging_obj=logger, + ) + return litellm.embedding( + model=model, + input=["hello"], + api_key="fake-key", + input_cost_per_token=rate, + output_cost_per_token=rate, + mock_response=[0.1], + litellm_logging_obj=logger, ) - entry: Final = litellm.model_cost[deployment_id] - assert entry["input_cost_per_token"] == 0.00000096 - assert entry["output_cost_per_token"] == 0.00000302 - assert litellm.model_cost[builtin_key] == builtin_row_before + def price(rate: float) -> float | None: + logger: Final = _request_logger(model, call_type) + response: Final = execute(logger, rate) + ready.wait(timeout=10) + return logger.response_cost_calculator(response, ignore_stamped_cost=True) + + with ThreadPoolExecutor(max_workers=2) as pool: + amounts: Final = tuple(pool.map(price, (0.001, 0.003))) + assert amounts[0] is not None and amounts[0] > 0 + assert amounts[1] == pytest.approx(amounts[0] * 3) + assert _snapshot_model_cost_entries((model, model.removeprefix("openai/"))) == before + + +def test_request_pricing_preserves_reasoning_and_cache_rates() -> None: + from litellm.types.utils import ModelResponse, Usage + + model: Final = "openai/gpt-4o-mini" + logger: Final = _request_logger(model, "completion") + litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + input_cost_per_token=0.01, + output_cost_per_token=0.02, + cache_read_input_token_cost=0.001, + output_cost_per_reasoning_token=0.04, + mock_response="hello", + litellm_logging_obj=logger, + ) + response: Final = ModelResponse( + model="gpt-4o-mini", + usage=Usage( + prompt_tokens=100, + completion_tokens=20, + prompt_tokens_details={"cached_tokens": 40}, + completion_tokens_details={"reasoning_tokens": 5}, + ), + ) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx( + 60 * 0.01 + 40 * 0.001 + 15 * 0.02 + 5 * 0.04 + ) + + +@pytest.mark.parametrize("explicit_null_mode", (False, True)) +def test_router_custom_token_prices_accept_missing_deployment_mode(explicit_null_mode: bool) -> None: + from litellm import Router + from litellm.types.utils import ModelResponse + + deployment_id: Final = "custom-pricing-without-mode" + keys: Final = ("gpt-4o-mini", "openai/gpt-4o-mini", deployment_id) + original: Final = _snapshot_model_cost_entries(keys) + model_info: Final = {"id": deployment_id, **({"mode": None} if explicit_null_mode else {})} + try: + router: Final = Router( + model_list=[ + { + "model_name": "priced-chat", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "fake-key", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "mock_response": "hello back", + }, + "model_info": model_info, + } + ], + num_retries=0, + ) + registered: Final = _snapshot_model_cost_entries(keys) + logger: Final = _request_logger("openai/gpt-4o-mini", "completion") + response: Final = router.completion( + model="priced-chat", messages=[{"role": "user", "content": "hello"}], litellm_logging_obj=logger + ) + assert isinstance(response, ModelResponse) + assert response.usage.prompt_tokens > 0 + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx( + response.usage.prompt_tokens * 0.001 + response.usage.completion_tokens * 0.002 + ) + assert _snapshot_model_cost_entries(keys) == registered finally: - _restore_model_cost_entries(model_cost_entries) - litellm.open_ai_chat_completion_models.intersection_update(openai_models_before) + _restore_model_cost_entries(original) + + +def test_changed_request_flat_rate_preserves_explicit_deployment_tier_prices() -> None: + from litellm.types.utils import ModelResponse, Usage + + deployment_id: Final = "request-price-variants" + original: Final = _snapshot_model_cost_entries((deployment_id,)) + variants: Final = {"input_cost_per_token_priority": 0.03, "output_cost_per_token_priority": 0.04} + try: + litellm.register_model( + { + deployment_id: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + **variants, + } + }, + persist_across_reloads=False, + ) + logger: Final = _request_logger("openai/gpt-4o-mini", "completion") + logger.update_environment_variables( + model="gpt-4o-mini", custom_llm_provider="openai", litellm_params={}, optional_params={} + ) + logger.set_request_pricing( + "gpt-4o-mini", + "openai", + {"input_cost_per_token": 0.05, "output_cost_per_token": 0.02, **variants}, + deployment_id, + ) + response: Final = ModelResponse( + model="gpt-4o-mini", + service_tier="priority", + usage=Usage(prompt_tokens=10, completion_tokens=4, total_tokens=14), + ) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx( + 10 * 0.03 + 4 * 0.04 + ) + finally: + _restore_model_cost_entries(original) + + +@pytest.mark.parametrize("call_type", ("completion", "embedding")) +def test_sdk_followup_prices_change_only_after_explicit_registration(call_type: str) -> None: + model: Final = f"openai/followup-{call_type}-prices" + original: Final = _snapshot_model_cost_entries((model, model.removeprefix("openai/"))) + models_before: Final = frozenset(litellm.open_ai_chat_completion_models) + logger: Final = _request_logger(model, call_type) + try: + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "chat" if call_type == "completion" else "embedding", + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0002, + } + }, + persist_across_reloads=False, + ) + + def request(prices: dict[str, float], *, logging_obj: Logging | None = None) -> object: + if call_type == "completion": + return litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="hello back", + litellm_logging_obj=logging_obj, + **prices, + ) + return litellm.embedding( + model=model, + input=["hello"], + api_key="fake-key", + mock_response=[0.1, 0.2], + litellm_logging_obj=logging_obj, + **prices, + ) + + first: Final = request({"input_cost_per_token": 0.001, "output_cost_per_token": 0.002}, logging_obj=logger) + second: Final = request({}) + assert logger.response_cost_calculator(first, ignore_stamped_cost=True) == pytest.approx( + 0.05 if call_type == "completion" else 0.01 + ) + assert litellm.completion_cost(completion_response=second, model=model, call_type=call_type) == pytest.approx( + 0.005 if call_type == "completion" else 0.001 + ) + assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001 + litellm.register_model( + {model: {"input_cost_per_token": 0.0003, "output_cost_per_token": 0.0006}}, + persist_across_reloads=False, + ) + third: Final = request({}) + assert litellm.completion_cost(completion_response=third, model=model, call_type=call_type) == pytest.approx( + 0.015 if call_type == "completion" else 0.003 + ) + assert logger.response_cost_calculator(first, ignore_stamped_cost=True) == pytest.approx( + 0.05 if call_type == "completion" else 0.01 + ) + finally: + _restore_model_cost_entries(original) + litellm.open_ai_chat_completion_models.intersection_update(models_before) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ("completion", "acompletion", "embedding", "aembedding")) +@pytest.mark.parametrize("custom_primary", (True, False)) +async def test_router_fallback_prices_belong_to_the_served_attempt( + monkeypatch: pytest.MonkeyPatch, call_type: str, custom_primary: bool +) -> None: + import httpx + import respx + from datetime import datetime, timezone + + model: Final = "openai/fallback-price-regression" + embedding: Final = "embedding" in call_type + keys: Final = (model, model.removeprefix("openai/"), "price-primary", "price-fallback") + original: Final = _snapshot_model_cost_entries(keys) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "cost_discount_config", {}) + monkeypatch.setattr(litellm, "cost_margin_config", {}) + try: + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "embedding" if embedding else "chat", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + } + }, + persist_across_reloads=False, + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": name, + "litellm_params": { + "model": model, + "api_key": "fake-key", + "api_base": f"https://{name}.example/v1", + **({"input_cost_per_token": rate, "output_cost_per_token": rate} if rate is not None else {}), + }, + "model_info": {"id": f"price-{name}"}, + } + for name, rate in ( + ("primary", 0.0 if custom_primary else None), + ("fallback", None if custom_primary else 0.003), + ) + ], + fallbacks=[{"primary": ["fallback"]}], + num_retries=0, + ) + registered: Final = _snapshot_model_cost_entries(keys) + logger: Final = Logging( + model=model, + messages=[], + stream=False, + call_type=call_type, + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="fallback-pricing", + function_id="fallback-pricing", + ) + endpoint: Final = "embeddings" if embedding else "chat/completions" + body: Final = ( + { + "object": "list", + "model": model.removeprefix("openai/"), + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "usage": {"prompt_tokens": 10, "total_tokens": 10}, + } + if embedding + else { + "id": "fallback-result", + "object": "chat.completion", + "created": 1, + "model": model.removeprefix("openai/"), + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + ) + with respx.mock(assert_all_called=True) as wire: + primary: Final = wire.post(f"https://primary.example/v1/{endpoint}").mock( + return_value=httpx.Response(429, json={"error": {"message": "rate limit", "type": "rate_limit_error"}}) + ) + fallback: Final = wire.post(f"https://fallback.example/v1/{endpoint}").mock( + return_value=httpx.Response(200, json=body) + ) + request: Final = {"model": "primary", "litellm_logging_obj": logger} + if call_type == "acompletion": + response = await router.acompletion(**request, messages=[{"role": "user", "content": "probe"}]) + elif call_type == "completion": + response = router.completion(**request, messages=[{"role": "user", "content": "probe"}]) + elif call_type == "aembedding": + response = await router.aembedding(**request, input=["probe"]) + else: + response = router.embedding(**request, input=["probe"]) + expected: Final = (0.01 if embedding else 0.02) if custom_primary else (0.03 if embedding else 0.045) + assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(expected) + assert response._hidden_params["response_cost"] == pytest.approx(expected) + assert primary.called + assert fallback.call_count == 1 + assert _snapshot_model_cost_entries(keys) == registered + finally: + _restore_model_cost_entries(original) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 5cf872f4ea7..add903ad561 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2333,7 +2333,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None: @respx.mock @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") -def test_completion_registers_test_owned_model_prices() -> None: +def test_completion_keeps_custom_prices_scoped_to_the_request() -> None: model: Final = "migration-owned-pricing-model" messages: Final = [{"role": "user", "content": "price control"}] litellm.register_model( @@ -2376,8 +2376,8 @@ def test_completion_registers_test_owned_model_prices() -> None: ) assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages} - assert litellm.model_cost[model]["input_cost_per_token"] == 0.001 - assert litellm.model_cost[model]["output_cost_per_token"] == 0.002 + assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001 + assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002 assert response._hidden_params["response_cost"] == pytest.approx(0.018)