This commit is contained in:
joshua-berri 2026-11-02 15:10:26 +08:00 • committed by GitHub
commit 402cde5d83
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 4147 additions and 341 deletions

View file

@ -0,0 +1,8 @@
SET LOCAL lock_timeout = '10s';
ALTER TABLE "LiteLLM_AccountingEvent"
ADD COLUMN IF NOT EXISTS "compilation_state" TEXT NOT NULL DEFAULT 'pending',
ADD COLUMN IF NOT EXISTS "compilation_error" TEXT,
ADD COLUMN IF NOT EXISTS "compilation_retry_at" TIMESTAMP(3);
CREATE INDEX IF NOT EXISTS "LiteLLM_AccountingEvent_kind_compilation_state_recorded_at_idx"
ON "LiteLLM_AccountingEvent"("kind", "compilation_state", "recorded_at");

View file

@ -2111,8 +2111,12 @@ model LiteLLM_AccountingEvent {
occurred_at DateTime
recorded_at DateTime @default(now())
payload Json
compilation_state String @default("pending")
compilation_error String?
compilation_retry_at DateTime?
@@index([recorded_at])
@@index([kind, compilation_state, recorded_at])
@@index([kind, source_event_id])
}

View file

@ -26,6 +26,12 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
from litellm.litellm_core_utils.llm_cost_calc.pricing_context import (
pricing_discounts,
pricing_margins,
retained_model_info,
retained_models,
)
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
@ -379,7 +385,9 @@ def _per_second_pricing_cost(
audio_seconds: float = 0.0,
) -> tuple[float, float] | None:
try:
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
model_info: Final = retained_model_info(model, custom_llm_provider) or cached_get_model_info_helper(
model=model, custom_llm_provider=custom_llm_provider
)
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
return None
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
@ -538,7 +546,7 @@ def cost_per_token(
if response_cost is not None:
return response_cost[0], response_cost[1]
model_cost_ref: Final = litellm.model_cost
model_cost_ref: Final = retained_models() or litellm.model_cost
# Only callers that explicitly pass `custom_llm_provider` get the
# dedup/prefix-join treatment. When provider is omitted, preserve legacy
# behavior: `model_with_provider` stays equal to the raw `model` string
@ -600,9 +608,11 @@ def cost_per_token(
model = model_without_prefix
if is_decisions_call_type(call_type):
decisions_model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
decisions_model_info: Final = retained_model_info(model, custom_llm_provider) or cached_get_model_info_helper(
model=model, custom_llm_provider=custom_llm_provider
)
decisions_overrides: Final = decisions_pricing_fields(
litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded
(retained_models() or litellm.model_cost).get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded
)
if decisions_overrides:
return generic_cost_per_token(
@ -623,7 +633,9 @@ def cost_per_token(
)
if lyria_generation_cost is not None:
return 0.0, lyria_generation_cost
speech_model_info: Final = cached_get_model_info_helper(
speech_model_info: Final = retained_model_info(
model_without_prefix, custom_llm_provider
) or cached_get_model_info_helper(
model=model_without_prefix,
custom_llm_provider=custom_llm_provider,
preserve_unset_costs=True,
@ -808,7 +820,9 @@ def cost_per_token(
service_tier=service_tier,
)
else:
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
model_info: Final = retained_model_info(model, custom_llm_provider) or cached_get_model_info_helper(
model=model, custom_llm_provider=custom_llm_provider
)
if _has_token_or_tiered_pricing(model_info):
return generic_cost_per_token(
model=model,
@ -995,8 +1009,8 @@ def select_model_name_for_cost_calc(
)
if custom_pricing is True:
if router_model_id is not None and router_model_id in litellm.model_cost:
entry: Final = litellm.model_cost[router_model_id]
if router_model_id is not None and router_model_id in (retained_models() or litellm.model_cost):
entry: Final = (retained_models() or litellm.model_cost)[router_model_id]
if _cost_map_entry_prices_anything(entry):
return_model = router_model_id
else:
@ -1075,14 +1089,16 @@ def _strip_unregistered_leading_segments(model: str, region_name: str | None) ->
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
resolves downstream (custom-priced router ids) or no stripped candidate is registered (#38069)."""
segments: Final = model.split("/")
if "/".join(segments[1:]) in litellm.model_cost:
if "/".join(segments[1:]) in (retained_models() or litellm.model_cost):
return model
head_len: Final = 2 if region_name is not None and len(segments) > 2 and segments[1] == region_name else 1
head: Final = "/".join(segments[:head_len])
tail: Final = segments[head_len:]
strippable: Final = next((index for index, segment in enumerate(tail) if segment in LlmProvidersSet), len(tail))
candidates: Final = (f"{head}/{'/'.join(tail[start:])}" for start in range(min(strippable, len(tail) - 1) + 1))
return next((candidate for candidate in candidates if candidate in litellm.model_cost), model)
return next(
(candidate for candidate in candidates if candidate in (retained_models() or litellm.model_cost)), model
)
@lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
@ -1348,9 +1364,7 @@ def _apply_cost_discount(
Tuple of (final_cost, discount_percent, discount_amount)
"""
original_cost: Final = base_cost
resolved_discount: Final = resolve_cost_discount(
litellm.cost_discount_config, custom_llm_provider, model, region_name
)
resolved_discount: Final = resolve_cost_discount(pricing_discounts(), custom_llm_provider, model, region_name)
discount_percent: Final = 0.0 if resolved_discount is None else resolved_discount
discount_amount: Final = original_cost * discount_percent
@ -1389,12 +1403,12 @@ def _apply_cost_margin(
# Get margin config - check provider-specific first, then global
margin_config = None
if custom_llm_provider and custom_llm_provider in litellm.cost_margin_config:
margin_config = litellm.cost_margin_config[custom_llm_provider]
if custom_llm_provider and custom_llm_provider in pricing_margins():
margin_config = pricing_margins()[custom_llm_provider]
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug("Found provider-specific margin config for %s: %s", custom_llm_provider, margin_config)
elif "global" in litellm.cost_margin_config:
margin_config = litellm.cost_margin_config["global"]
elif "global" in pricing_margins():
margin_config = pricing_margins()["global"]
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug("Using global margin config: %s", margin_config)
else:
@ -1402,7 +1416,7 @@ def _apply_cost_margin(
verbose_logger.debug(
"No margin config found. Provider: %s, Available configs: %s",
custom_llm_provider,
list(litellm.cost_margin_config.keys()),
list(pricing_margins().keys()),
)
if margin_config is not None:
@ -1995,7 +2009,7 @@ def completion_cost(
video_resolution=video_resolution,
)
elif call_type in _SPEECH_CALL_TYPES:
prompt_characters = _speech_billed_characters(
prompt_characters = speech_billed_characters(
completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None,
prompt,
)
@ -2159,12 +2173,14 @@ def completion_cost(
# Calculate cost based on prompt_tokens, completion_tokens
if (
"togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
) and not has_together_registry_pricing(model, litellm.model_cost):
) and not has_together_registry_pricing(model, (retained_models() or litellm.model_cost)):
model = get_model_params_and_category(model, call_type=CallTypes(call_type))
# replicate llms are calculate based on time for request running
# see https://replicate.com/pricing
elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost:
elif (model in litellm.replicate_models or "replicate" in model) and model not in (
retained_models() or litellm.model_cost
):
# for unmapped replicate model, default to replicate's time tracking logic
return get_replicate_completion_pricing(completion_response, response_time_ms)
@ -2261,7 +2277,7 @@ def completion_cost(
_final_cost += sum(additional_costs.values())
original_cost = _final_cost
if litellm.cost_discount_config:
if pricing_discounts():
(
_final_cost,
discount_percent,
@ -2277,7 +2293,7 @@ def completion_cost(
discount_amount = 0.0
# Apply margin from module-level config if configured
if litellm.cost_margin_config:
if pricing_margins():
(
_final_cost,
margin_percent,
@ -2352,15 +2368,15 @@ def completion_cost(
def get_response_cost_from_hidden_params(
hidden_params: dict | BaseModel,
hidden_params: Mapping[str, object] | BaseModel,
) -> float | None:
if isinstance(hidden_params, BaseModel):
_hidden_params_dict = hidden_params.model_dump()
else:
_hidden_params_dict = hidden_params
additional_headers: Final = _hidden_params_dict.get("additional_headers", {})
if additional_headers and "llm_provider-x-litellm-response-cost" in additional_headers:
additional_headers: Final = as_str_mapping(_hidden_params_dict.get("additional_headers"))
if additional_headers is not None and "llm_provider-x-litellm-response-cost" in additional_headers:
return _billable_header_cost(additional_headers["llm_provider-x-litellm-response-cost"])
return None
@ -2511,7 +2527,7 @@ def response_cost_calculator(
raise e
def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int:
def speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int:
reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None
return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt)
@ -2629,7 +2645,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, .
def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
raw_entry: Final = litellm.model_cost.get(key)
raw_entry: Final = (retained_models() or litellm.model_cost).get(key)
return raw_entry if isinstance(raw_entry, Mapping) else None
@ -2908,6 +2924,46 @@ def transcription_cost(model: str, custom_llm_provider: str | None, duration: fl
return openai_cost_per_second(model=model, custom_llm_provider=custom_llm_provider, duration=duration)
def image_cost_model_candidates(
model: str, custom_llm_provider: str | None, quality: str | None, size: str | None
) -> tuple[str | None, ...]:
# Standardize size format to use "-x-"
size_str: str = size or "1024-x-1024"
size_str = size_str.replace("x", "-x-") if "x" in size_str and "-x-" not in size_str else size_str
# Build model names for cost lookup
base_model_name = f"{size_str}/{model}"
model_name_without_custom_llm_provider: str | None = None
if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"):
model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "")
base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}"
model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name
provider_first_model_name_with_quality: Final = (
f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}"
if quality and custom_llm_provider
else None
)
# gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family
model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}"
verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name)
model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}"
model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider
return (
model_name_with_quality,
provider_first_model_name_with_quality,
base_model_name,
model_name_with_v2_quality,
model_with_quality_without_provider,
model_without_provider,
model,
model_name_without_custom_llm_provider,
)
def default_image_cost_calculator(
model: str,
custom_llm_provider: str | None = None,
@ -2934,51 +2990,23 @@ def default_image_cost_calculator(
Raises:
Exception: If model pricing not found in cost map
"""
# Standardize size format to use "-x-"
size_str: str = size or "1024-x-1024"
size_str = size_str.replace("x", "-x-") if "x" in size_str and "-x-" not in size_str else size_str
# Parse dimensions
height, width = map(int, size_str.split("-x-"))
# Build model names for cost lookup
base_model_name = f"{size_str}/{model}"
model_name_without_custom_llm_provider: str | None = None
if custom_llm_provider and model.startswith(f"{custom_llm_provider}/"):
model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "")
base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}"
model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name
provider_first_model_name_with_quality: Final = (
f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}"
if quality and custom_llm_provider
else None
)
# gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family
model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}"
verbose_logger.debug("Looking up cost for models: %s, %s", model_name_with_quality, base_model_name)
model_without_provider: Final = f"{size_str}/{model.split('/')[-1]}"
model_with_quality_without_provider = f"{quality}/{model_without_provider}" if quality else model_without_provider
models_to_check: Final = (
model_name_with_quality,
provider_first_model_name_with_quality,
base_model_name,
model_name_with_v2_quality,
model_with_quality_without_provider,
model_without_provider,
model,
model_name_without_custom_llm_provider,
)
size_str: Final = (size or "1024x1024").replace("-x-", "x")
height, width = map(int, size_str.split("x"))
models_to_check: Final = image_cost_model_candidates(model, custom_llm_provider, quality, size)
matched_model: Final = next(
(_model for _model in models_to_check if _model is not None and _model in litellm.model_cost), None
(
_model
for _model in models_to_check
if _model is not None and _model in (retained_models() or litellm.model_cost)
),
None,
)
if matched_model is None and model_info is None:
raise Exception(f"Model not found in cost map. Tried checking {models_to_check}")
shared_cost_info: Final = litellm.model_cost[matched_model] if matched_model is not None else None
shared_cost_info: Final = (
(retained_models() or litellm.model_cost)[matched_model] if matched_model is not None else None
)
price_tables: Final = tuple(table for table in (model_info, shared_cost_info) if table is not None)
image_count: Final = n if n is not None else 1
unit_counts: Final = (
@ -3016,7 +3044,7 @@ def default_video_cost_calculator(
custom_llm_provider (Optional[str]): Custom LLM provider
model_info (Optional[ModelInfo]): Deployment-level model info containing
custom video pricing. When provided, used before falling back to
the global litellm.model_cost lookup.
the effective model cost lookup.
video_resolution (Optional[str]): From usage (e.g. ``720p``, ``1080p``) for tiered per-second pricing.
Returns:
@ -3026,9 +3054,9 @@ def default_video_cost_calculator(
Exception: If model pricing not found in cost map
"""
# Use custom model_info pricing if provided (deployment-specific pricing)
cost_info: dict | None = None
cost_info: ModelInfo | None = None
if model_info is not None:
cost_info = dict(model_info)
cost_info = model_info
else:
# Build model names for cost lookup
base_model_name = model
@ -3049,15 +3077,15 @@ def default_video_cost_calculator(
model_name_without_custom_llm_provider,
]
for _model in models_to_check:
if _model is not None and _model in litellm.model_cost:
cost_info = litellm.model_cost[_model]
if _model is not None and _model in (retained_models() or litellm.model_cost):
cost_info = (retained_models() or litellm.model_cost)[_model]
break
# If still not found, try with custom_llm_provider prefix
if cost_info is None and custom_llm_provider:
prefixed_model: Final = f"{custom_llm_provider}/{model}"
if prefixed_model in litellm.model_cost:
cost_info = litellm.model_cost[prefixed_model]
if prefixed_model in (retained_models() or litellm.model_cost):
cost_info = (retained_models() or litellm.model_cost)[prefixed_model]
if cost_info is None:
raise Exception(f"Model not found in cost map for model={model}")
@ -3663,9 +3691,9 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str)
"""
resolved: Final = _cost_map_model_info(model_name, custom_llm_provider)
entries: Final = (
litellm.model_cost.get(resolved.get("key")) if resolved is not None else None,
litellm.model_cost.get(model_name),
litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"),
(retained_models() or litellm.model_cost).get(resolved.get("key")) if resolved is not None else None,
(retained_models() or litellm.model_cost).get(model_name),
(retained_models() or litellm.model_cost).get(f"{custom_llm_provider}/{model_name}"),
)
return any(entry is not None and _cost_map_entry_prices_anything(entry) for entry in entries)
@ -3910,11 +3938,11 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
"""First of ``keys`` this entry prices, read off the raw model cost entry
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
if info is None:
return None
declared: Final = litellm.model_cost.get(info.get("key"))
declared: Final = (retained_models() or litellm.model_cost).get(info.get("key"))
if declared is None:
return None
return next(

View file

@ -14,6 +14,7 @@ import sys
import time
import traceback
from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
from contextlib import nullcontext
from dataclasses import dataclass
from datetime import datetime as dt_object
from enum import Enum
@ -25,7 +26,13 @@ from httpx import Response, Timeout
from pydantic import BaseModel, JsonValue, TypeAdapter
import litellm
from litellm._internal_context import in_mirrored_call, outside_mirrored_call, post_response_phase
from litellm._internal_context import (
current_billing_time,
in_mirrored_call,
outside_mirrored_call,
pinned_billing_time,
post_response_phase,
)
from litellm._logging import (
is_debugging_on,
redact_string,
@ -54,9 +61,11 @@ from litellm.constants import (
from litellm.cost_calculator import (
RealtimeAPITokenUsageProcessor,
ResponsesWebSocketTokenUsageProcessor,
get_response_cost_from_hidden_params,
get_usage_object,
pricing_entry_for_cost_calc,
select_model_name_for_cost_calc,
speech_billed_characters,
)
from litellm.exceptions import (
BudgetExceededError,
@ -78,6 +87,7 @@ from litellm.litellm_core_utils.classifier_logging import (
is_classifier_call,
)
from litellm.litellm_core_utils.core_helpers import (
billed_usage_cost,
finite_cost,
get_provider_response_headers_from_hidden_params,
is_expected_client_error,
@ -112,6 +122,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
object_mapping,
with_guardrail_entry,
)
from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
@ -666,6 +677,10 @@ class UsageCaptureObservation:
return cls(serialized, provenance, response_cost)
_REQUEST_MODEL_INFO: Final = TypeAdapter(ModelInfo)
_REQUEST_PRICING_MODELS: Final = TypeAdapter(dict[str, ModelInfo])
class Logging(LiteLLMLoggingBaseClass):
usage_capture_callback: ClassVar[
Callable[
@ -703,6 +718,8 @@ class Logging(LiteLLMLoggingBaseClass):
logfireLogger, \
prometheusLogger, \
slack_app
cost_billing_time: datetime.datetime | None = None
request_pricing: RetainedPricing | None = None
custom_pricing: bool = False
custom_cost_per_token: CostPerToken | None = None
custom_cost_per_second: float | None = None
@ -2107,6 +2124,70 @@ class Logging(LiteLLMLoggingBaseClass):
if margin_total_amount is not None:
self.cost_breakdown["margin_total_amount"] = margin_total_amount
def _request_pricing_model_info(
self, model: str, provider: str, overrides: Mapping[str, object], deployment_id: str | None
) -> ModelInfo:
for candidate in (deployment_id, f"{provider}/{model}"):
if candidate is None:
continue
try:
return litellm.get_model_info(model=candidate, custom_llm_provider=provider)
except Exception: # noqa: BLE001 # unmapped SDK models can supply their complete pricing
verbose_logger.debug("No base pricing for request override candidate %s", candidate)
return _REQUEST_MODEL_INFO.validate_python(
{
"key": deployment_id or f"{provider}/{model}",
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
"input_cost_per_token": None,
"output_cost_per_token": None,
"litellm_provider": provider,
"mode": overrides.get(
"mode", "embedding" if "embedding" in str(self.model_call_details.get("call_type", "")) else "chat"
),
"supported_openai_params": None,
}
)
def set_request_pricing(
self,
model: str,
provider: str,
overrides: Mapping[str, object] | None,
deployment_id: str | None = None,
) -> None:
from litellm.utils import model_info_with_pricing_overrides
if overrides is None:
self.request_pricing = None
return
base: Final = self._request_pricing_model_info(model, provider, overrides, deployment_id)
normalized: Final = _REQUEST_MODEL_INFO.validate_python(
{
**base,
"mode": base.get("mode")
or ("embedding" if "embedding" in str(self.model_call_details.get("call_type", "")) else "chat"),
}
)
effective: Final = model_info_with_pricing_overrides(
normalized,
{
name: value
for name, value in overrides.items()
if deployment_id is None
or name not in {"input_cost_per_token", "output_cost_per_token"}
or normalized.get(name) != value
},
)
keys: Final = (model, f"{provider}/{model}", deployment_id or model, effective["key"])
self.request_pricing = RetainedPricing(
models_json=_REQUEST_PRICING_MODELS.dump_json({name: effective for name in keys}).decode(),
discounts_json=json.dumps(litellm.cost_discount_config),
margins_json=json.dumps(litellm.cost_margin_config),
allow_live_models=True,
)
def response_cost_calculator(
self,
result: object,
@ -2114,6 +2195,26 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_model_name: str | None = None,
router_model_id: str | None = None,
ignore_stamped_cost: bool = False,
custom_pricing_override: bool | None = None,
) -> float | None:
context: Final = (
retained_pricing(self.request_pricing)
if self.request_pricing is not None and not self.spilled_over_ptu_for(result)
else nullcontext()
)
with context, pinned_billing_time(self.billing_time_for_cost()):
return self._calculate_response_cost(
result, cache_hit, litellm_model_name, router_model_id, ignore_stamped_cost, custom_pricing_override
)
def _calculate_response_cost(
self,
result: object,
cache_hit: bool | None = None,
litellm_model_name: str | None = None,
router_model_id: str | None = None,
ignore_stamped_cost: bool = False,
custom_pricing_override: bool | None = None,
) -> float | None:
"""
Calculate response cost using result + logging object variables.
@ -2176,7 +2277,9 @@ class Logging(LiteLLMLoggingBaseClass):
router_model_id = self.get_router_model_id()
## RESPONSE COST ##
custom_pricing: Final = self.custom_pricing_for(priced_result)
custom_pricing: Final = (
self.custom_pricing_for(priced_result) if custom_pricing_override is None else custom_pricing_override
)
prompt: Final = self._prompt_for_cost_calculation()
@ -2404,7 +2507,9 @@ class Logging(LiteLLMLoggingBaseClass):
def custom_pricing_for(self, result: object) -> bool:
if self.spilled_over_ptu_for(result):
return False
return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None))
return self.request_pricing is not None or use_custom_pricing_for_model(
litellm_params=getattr(self, "litellm_params", None)
)
def _prompt_for_cost_calculation(self) -> str:
"""
@ -2965,6 +3070,8 @@ class Logging(LiteLLMLoggingBaseClass):
metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
return {
"id": self.litellm_call_id,
"model": self.model,
"results": result,
"usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")}
if metered is not None
else None,
@ -2974,15 +3081,31 @@ class Logging(LiteLLMLoggingBaseClass):
)
return normalized if assembled is None else assembled
def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]:
from litellm.utils import count_characters
def billing_time_for_cost(self) -> datetime.datetime:
if self.cost_billing_time is None:
self.cost_billing_time = current_billing_time()
return self.cost_billing_time
def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]:
return TypeAdapter(dict[str, JsonValue]).validate_python(
{
"response_ms": getattr(result, "_response_ms", None),
"provider_reported_cost": self._provider_reported_cost
if self._provider_reported_cost is not None
else get_response_cost_from_hidden_params(dict(get_hidden_params(result) or {})),
"provider_usage_cost": billed_usage_cost(result, self.custom_llm_provider),
"current_billing_time": self.billing_time_for_cost().isoformat(),
"custom_pricing": self.custom_pricing_for(result),
"custom_cost_per_token": self.custom_cost_per_token,
"custom_cost_per_second": self.custom_cost_per_second,
"prompt_characters": count_characters(self._prompt_for_cost_calculation())
"tool_response_cost": self.model_call_details.get("response_cost")
if self.model_call_details.get("call_type")
in (CallTypes.call_mcp_tool.value, CallTypes.send_message.value, CallTypes.asend_message.value)
else None,
"prompt_characters": speech_billed_characters(
result if isinstance(result, HttpxBinaryResponseContent) else None,
self._prompt_for_cost_calculation(),
)
if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value)
else None,
}
@ -7838,14 +7961,20 @@ def get_standard_logging_object_payload(
timing_metrics: Final = getattr(logging_obj, "response_timing_metrics", None) or {}
clean_hidden_params["litellm_overhead_time_ms"] = timing_metrics.get("litellm_overhead_time_ms")
model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information(
base_model=base_model,
custom_pricing=custom_pricing,
custom_llm_provider=kwargs.get("custom_llm_provider"),
init_response_obj=init_response_obj,
api_base=litellm_params.get("api_base"),
router_model_id=logging_obj.get_router_model_id(),
pricing_context: Final = (
retained_pricing(logging_obj.request_pricing)
if logging_obj.request_pricing is not None and not spilled_over
else nullcontext()
)
with pricing_context:
model_cost_information: Final = StandardLoggingPayloadSetup.get_model_cost_information(
base_model=base_model,
custom_pricing=custom_pricing,
custom_llm_provider=kwargs.get("custom_llm_provider"),
init_response_obj=init_response_obj,
api_base=litellm_params.get("api_base"),
router_model_id=logging_obj.get_router_model_id(),
)
error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
metadata=metadata,

View file

@ -0,0 +1,84 @@
from collections import ChainMap
from collections.abc import Generator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Final, cast # noqa: TID251 # live catalog has no typed boundary yet
from pydantic import TypeAdapter
from litellm.types.utils import ModelInfo
_MODELS: Final = TypeAdapter(dict[str, ModelInfo])
_DISCOUNTS: Final = TypeAdapter(dict[str, float])
_MARGINS: Final = TypeAdapter(dict[str, float | dict[str, float]])
@dataclass(frozen=True, slots=True)
class RetainedPricing:
models_json: str
discounts_json: str
margins_json: str
allow_live_models: bool = False
def models(self) -> Mapping[str, ModelInfo]:
return _MODELS.validate_json(self.models_json)
_CURRENT: Final[ContextVar[RetainedPricing | None]] = ContextVar("retained_pricing", default=None)
@contextmanager
def retained_pricing(pricing: RetainedPricing) -> Generator[None]:
token: Final = _CURRENT.set(pricing)
try:
yield
finally:
_CURRENT.reset(token)
def retained_model_info(model: str, provider: str | None) -> ModelInfo | None:
context: Final = _CURRENT.get()
if context is None:
return None
models: Final = context.models()
result: Final = models.get(f"{provider}/{model}") or models.get(model)
if result is None and not context.allow_live_models:
raise ValueError(f"Pricing snapshot has no retained model {model}")
return result
def retained_models() -> Mapping[str, ModelInfo] | None:
import litellm
context: Final = _CURRENT.get()
if context is None:
return None
if not context.allow_live_models:
return context.models()
catalog: Final = cast( # cast-ok: legacy catalog getter has an untyped dict signature
dict[str, ModelInfo], vars(litellm)["model_cost"]
)
return ChainMap(dict(context.models()), catalog)
def pricing_discounts() -> Mapping[str, float]:
import litellm
context: Final = _CURRENT.get()
return (
litellm.cost_discount_config
if context is None or context.allow_live_models
else _DISCOUNTS.validate_json(context.discounts_json)
)
def pricing_margins() -> Mapping[str, float | Mapping[str, float]]:
import litellm
context: Final = _CURRENT.get()
return (
litellm.cost_margin_config
if context is None or context.allow_live_models
else _MARGINS.validate_json(context.margins_json)
)

View file

@ -149,6 +149,19 @@ def _requested_image_size(optional_params: Mapping[str, object] | None) -> str |
return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
def image_response_size_and_quality(
response: ImageResponse,
optional_params: Mapping[str, object] | None = None,
*,
size: str | None = None,
quality: str | None = None,
) -> tuple[str, str]:
return (
size or response.size or _requested_image_size(optional_params) or "1024-x-1024",
quality or response.quality or _requested_image_param(optional_params, "quality") or "standard",
)
def get_web_search_requests(server_tool_use: object) -> int | None:
"""
Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value
@ -2187,11 +2200,8 @@ class CostCalculatorUtils:
cost_calculator as vertex_ai_image_cost_calculator,
)
resolved_size: Final = (
size or completion_response.size or _requested_image_size(optional_params) or "1024-x-1024"
)
resolved_quality: Final = (
quality or completion_response.quality or _requested_image_param(optional_params, "quality") or "standard"
resolved_size, resolved_quality = image_response_size_and_quality(
completion_response, optional_params, size=size, quality=quality
)
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
pricing: Final = deployment_pricing(model_info)

View file

@ -1316,7 +1316,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o
)
from litellm.responses.bridge_check import (
bridge_reasoning_effort,
keep_registered_mode,
str_keyed_mapping,
)
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
@ -1472,50 +1471,26 @@ def _record_agentic_loop_params(
)
def _register_custom_pricing_for_request(
def _set_request_pricing(
logging_obj: object,
model: str,
custom_llm_provider: str,
kwargs: dict,
model_info: dict | None,
) -> None:
"""Register per-request custom pricing in litellm.model_cost.
Router-originated requests (identified by the deployment id the router puts
in metadata) get their full pricing registered under that unique id only;
the shared ``{provider}/{model}`` key receives the entry with pricing fields
stripped, mirroring Router._create_deployment. This keeps one deployment's
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
pricing used by sibling deployments of the same backend model. Direct SDK
calls keep the legacy behavior of registering the shared key with pricing.
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
deployment registration, since a call routes by its own ``model_info``.
"""
entry: Final = _build_custom_pricing_entry(
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
shared_key: Final = f"{custom_llm_provider}/{model}"
deployment_id: Final = _get_router_deployment_id(kwargs)
if deployment_id is None:
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
if not isinstance(logging_obj, LiteLLMLoggingObj):
return
shared_entry: Final = keep_registered_mode(
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
custom_pricing: Final = (
(kwargs.get("input_cost_per_token") is not None and kwargs.get("output_cost_per_token") is not None)
or kwargs.get("input_cost_per_second") is not None
or kwargs.get("output_cost_per_second") is not None
or kwargs.get("cost_per_second") is not None
)
litellm.register_model(
{deployment_id: entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
add_to_provider_model_sets=False,
)
litellm.register_model(
{shared_key: shared_entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
add_to_provider_model_sets=False,
logging_obj.set_request_pricing(
model,
custom_llm_provider,
_build_custom_pricing_entry(custom_llm_provider, kwargs, model_info) if custom_pricing else None,
_get_router_deployment_id(kwargs),
)
@ -5973,19 +5948,13 @@ def completion(
"aws_region_name", None
) # support region-based pricing for bedrock
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if (
(input_cost_per_token is not None and output_cost_per_token is not None)
or input_cost_per_second is not None
or output_cost_per_second is not None
or cost_per_second is not None
):
_register_custom_pricing_for_request(
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
_set_request_pricing(
logging_obj=litellm_logging_obj,
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
custom_prompt_dict = {}
if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token:
@ -6992,12 +6961,6 @@ def embedding(
headers.update(proxy_headers)
except Exception as e:
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
### CUSTOM MODEL COST ###
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
cost_per_second: Final = kwargs.get("cost_per_second", None)
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
openai_params: Final = [
"user",
"dimensions",
@ -7044,19 +7007,13 @@ def embedding(
**non_default_params,
)
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if (
(input_cost_per_token is not None and output_cost_per_token is not None)
or input_cost_per_second is not None
or output_cost_per_second is not None
or cost_per_second is not None
):
_register_custom_pricing_for_request(
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=kwargs.get("model_info"),
)
_set_request_pricing(
logging_obj=litellm_logging_obj,
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=kwargs.get("model_info"),
)
logging: Final[LiteLLMLoggingObj] = litellm_logging_obj
logging.update_environment_variables(

View file

@ -11,7 +11,17 @@ from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from pydantic import AwareDatetime, ConfigDict, JsonValue, TypeAdapter
from typing_extensions import assert_never
from litellm.accounting import CostEvent, DeliveryTarget, EventKind, InboundUsageEvent, OutboundUsageEvent
from litellm.accounting import (
CostEvent,
DeliveryTarget,
EventKind,
InboundUsageEvent,
Metric,
MetricName,
OutboundUsageEvent,
PricingSnapshot,
Unpriced,
)
from litellm.proxy.db.db_span import db_span
from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
@ -143,6 +153,40 @@ SELECT records.event_id, destination.value::"AccountingDeliveryTarget", 1
FROM jsonb_to_recordset($1::jsonb) AS records(event_id text, targets jsonb)
CROSS JOIN LATERAL jsonb_array_elements_text(records.targets) AS destination
"""
_LOAD_SOURCE: Final = """
SELECT payload::text AS payload, schema_version::text AS schema_version
FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 AND kind = 'outbound_usage' FOR UPDATE
"""
_LOAD_COST: Final = """
SELECT payload::text AS payload, schema_version::text AS schema_version, payload_hash
FROM "LiteLLM_AccountingEvent" WHERE source_event_id = $1 AND kind = 'cost'
"""
_PENDING_COMPILATION: Final = """
SELECT payload::text AS payload FROM "LiteLLM_AccountingEvent"
WHERE kind = 'outbound_usage' AND compilation_state = 'pending'
AND (compilation_retry_at IS NULL OR compilation_retry_at <= CURRENT_TIMESTAMP)
ORDER BY compilation_retry_at NULLS FIRST, recorded_at, event_id LIMIT $1::int
"""
_COMPILATION_RESULT: Final = """
UPDATE "LiteLLM_AccountingEvent" SET compilation_state = $2, compilation_error = $3, compilation_retry_at = NULL
WHERE event_id = $1 AND kind = 'outbound_usage'
"""
_RETRY_COMPILATION: Final = """
UPDATE "LiteLLM_AccountingEvent"
SET compilation_error = $2, compilation_retry_at = CURRENT_TIMESTAMP + INTERVAL '30 seconds'
WHERE event_id = $1 AND kind = 'outbound_usage' AND compilation_state = 'pending'
"""
async def _unpriced(db: Prisma, event_id: str, reason: str) -> Unpriced:
await db.execute_raw(
_COMPILATION_RESULT, event_id, "excluded" if reason == "not_charge_owner" else "unpriced", reason
)
return Unpriced(usage_event_id=event_id, reason=reason)
_COMPLETED_FAMILY: Final = """
root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage')
AND root.recorded_at < $1::timestamp
@ -320,6 +364,79 @@ class AccountingLedger:
case _:
assert_never(result)
async def pending_compilation(self, limit: int = 100) -> tuple[OutboundUsageEvent, ...]:
if not 1 <= limit <= 1000:
raise ValueError("Compilation batch size must be between 1 and 1000")
async with self._transaction() as db:
rows: Final = _STRINGS.validate_python(await db.query_raw(_PENDING_COMPILATION, limit))
return tuple(
OutboundUsageEvent.model_validate_json(_json(_JSON.validate_json(row["payload"])["event"])) for row in rows
)
async def defer_compilation(self, event: OutboundUsageEvent) -> None:
async with self._transaction() as db:
await db.execute_raw(_RETRY_COMPILATION, event.event_id, "compilation_retryable")
async def compile_cost(
self,
event: OutboundUsageEvent,
price: Callable[[str], PricingSnapshot | Unpriced],
targets: tuple[DeliveryTarget, ...],
) -> CostEvent | Unpriced:
"""Serialize pure pricing with other compilers and commit its complete delivery set."""
validated: Final = OutboundUsageEvent.model_validate_json(event.model_dump_json())
async with self._transaction() as db:
sources: Final = _STRINGS.validate_python(await db.query_raw(_LOAD_SOURCE, validated.event_id))
if not sources:
return Unpriced(usage_event_id=validated.event_id, reason="usage_not_committed")
source: Final = sources[0]
if source["schema_version"] != "1":
return await _unpriced(db, validated.event_id, "unsupported_usage_schema")
payload: Final = _JSON.validate_json(source["payload"])
if payload.get("event") != validated.model_dump(mode="json"):
raise AccountingConflictError("Compilation input differs from the accepted usage event")
costs: Final = _STRINGS.validate_python(await db.query_raw(_LOAD_COST, validated.event_id))
if costs:
if costs[0]["schema_version"] != "1":
return await _unpriced(db, validated.event_id, "unsupported_cost_schema")
existing: Final = CostEvent.model_validate_json(
_json(_JSON.validate_json(costs[0]["payload"])["event"])
)
if existing.usage_event_id != validated.event_id:
raise AccountingConflictError("Committed cost does not match its source usage")
replay: Final = _prepare(existing, targets, costs[0]["payload"], 1, 1)
if replay.payload_hash != costs[0]["payload_hash"]:
raise AccountingConflictError("Compilation delivery set differs from the accepted cost")
await db.execute_raw(_COMPILATION_RESULT, validated.event_id, "compiled", None)
return existing
try:
priced: Final = price(source["payload"])
except ValueError:
return await _unpriced(db, validated.event_id, "invalid_pricing_inputs")
except Exception: # noqa: BLE001 # retain failed pricing for bounded retries without starving newer events
await db.execute_raw(_RETRY_COMPILATION, validated.event_id, "calculation_retryable")
return Unpriced(usage_event_id=validated.event_id, reason="calculation_retryable")
if isinstance(priced, Unpriced):
if priced.usage_event_id != validated.event_id:
raise AccountingConflictError("Unpriced result does not match its source usage")
return await _unpriced(db, validated.event_id, priced.reason)
cost: Final = CostEvent(
event_id=f"cost:{validated.event_id}",
usage_event_id=validated.event_id,
occurred_at=validated.ended_at,
dimensions=validated.dimensions,
charges=(Metric(name=MetricName.SPEND, value=priced.total_cost, unit="USD"),),
snapshot=priced,
)
record: Final = _prepare(
cost, targets, _json({"event": cost.model_dump(mode="json"), "pricing_inputs": payload}), 1, 1
)
accepted: Final = await _accept(db, record)
if accepted not in ("inserted", "duplicate"):
raise AccountingConflictError(f"Cannot commit compiled cost: {accepted}")
await db.execute_raw(_COMPILATION_RESULT, validated.event_id, "compiled", None)
return cost
async def purge_completed(
self,
*,

View file

@ -3256,11 +3256,15 @@ def _configure_usage_capture() -> None:
"Usage capture requires local Prisma writes and disabled collector, remote writer and search free tier"
)
from litellm.proxy.db.accounting_ledger import AccountingLedger
from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler
ledger: Final = AccountingLedger.for_client(prisma_client)
writer: Final = UsageBatchWriter(ledger.insert_usage_batch, ledger.load_usage_batch)
usage_capture.start(
writer.write, UsageCaptureSettings.model_validate(capture_settings), load=ledger.load_usage_batch
writer.write,
UsageCaptureSettings.model_validate(capture_settings),
load=ledger.load_usage_batch,
compiler=LoggingCostCompiler(ledger),
)
@ -11607,6 +11611,22 @@ class ProxyStartupEvent:
await cls._initialize_spend_tracking_background_jobs(scheduler=scheduler)
async def compile_accounting_costs() -> None:
from litellm.proxy.db.accounting_ledger import AccountingLedger
from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler
if usage_capture.meter is not None:
await LoggingCostCompiler(AccountingLedger.for_client(prisma_client)).compile_pending()
scheduler.add_job( # pyright: ignore[reportUnknownMemberType] # APScheduler has no typed add_job interface
compile_accounting_costs,
"interval",
seconds=5,
id="accounting_cost_compilation",
replace_existing=True,
max_instances=1,
)
cls._initialize_spend_capture_rate_check_job(
scheduler=scheduler,
proxy_logging_obj=proxy_logging_obj,

View file

@ -2111,8 +2111,12 @@ model LiteLLM_AccountingEvent {
occurred_at DateTime
recorded_at DateTime @default(now())
payload Json
compilation_state String @default("pending")
compilation_error String?
compilation_retry_at DateTime?
@@index([recorded_at])
@@index([kind, compilation_state, recorded_at])
@@index([kind, source_event_id])
}

View file

@ -0,0 +1,463 @@
import json
import math
from collections.abc import Callable, Mapping
from datetime import datetime
from decimal import Decimal
from typing import Final
from httpx import Response
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
from litellm._internal_context import pinned_billing_time
from litellm._logging import verbose_proxy_logger
from litellm.accounting import (
CostEvent,
DeliveryTarget,
DimensionName,
OutboundUsageEvent,
Outcome,
PricingSnapshot,
Unpriced,
)
from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost
from litellm.litellm_core_utils.hidden_params import set_hidden_param
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_cost_calc.finalization import finalize_cost
from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing
from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
from litellm.proxy.db.accounting_ledger import AccountingLedger
from litellm.proxy.spend_tracking.usage_meter import usage_delivery_targets
from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse
from litellm.types.rerank import RerankResponse
from litellm.types.utils import (
CallTypes,
CostBreakdown,
CostPerToken,
ImageResponse,
LiteLLMRealtimeStreamLoggingObject,
ModelInfo,
ModelResponse,
StandardBuiltInToolsParams,
Usage,
is_server_derived_pricing_key,
)
_FIELDS: Final = TypeAdapter(dict[str, JsonValue])
def _fields(value: object) -> Mapping[str, JsonValue]:
return _FIELDS.validate_python(value) if isinstance(value, dict) else {}
def _json(value: object) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False)
class _UsageEnvelope(BaseModel):
model_config = ConfigDict(frozen=True)
event: OutboundUsageEvent
usage: Mapping[str, JsonValue] | None = None
pricing: Mapping[str, JsonValue]
charge_eligible: bool
parent_request_id: str | None = None
finalized_cost: Mapping[str, JsonValue] | None = None
allocation: Mapping[str, JsonValue] | None = None
def price_usage(envelope: str) -> PricingSnapshot | Unpriced:
captured: Final = _UsageEnvelope.model_validate_json(envelope)
event: Final = captured.event
batch_summary: Final = event.call_type in {CallTypes.retrieve_batch, CallTypes.aretrieve_batch} and not (
captured.parent_request_id is not None and event.event_id.startswith(f"{captured.parent_request_id}:row:")
)
if not captured.charge_eligible or batch_summary:
return Unpriced(usage_event_id=event.event_id, reason="not_charge_owner")
response_inputs: Final = _fields(captured.pricing.get("response_inputs"))
rerank_usage: Final = event.call_type in {CallTypes.rerank, CallTypes.arerank} and any(
isinstance(value, int) and not isinstance(value, bool) and value >= 0
for name, value in _fields(response_inputs.get("billed_units")).items()
if name in {"search_units", "total_tokens"}
)
logged: Final = _fields(captured.pricing.get("logging"))
speech_characters: Final = logged.get("prompt_characters")
speech_usage: Final = (
event.call_type in {CallTypes.speech, CallTypes.aspeech}
and isinstance(speech_characters, int)
and not isinstance(speech_characters, bool)
and speech_characters >= 0
)
fixed_call: Final = event.call_type in {CallTypes.call_mcp_tool, CallTypes.send_message, CallTypes.asend_message}
realtime_events: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(
captured.pricing.get("realtime_events") or ()
)
realtime_usage: Final = event.call_type is CallTypes.arealtime and any(
bool(item.get("usage")) for item in realtime_events
)
image_inputs: Final = _fields(captured.pricing.get("image_response"))
if (
captured.usage is None
and not fixed_call
and not realtime_usage
and not speech_usage
and not rerank_usage
and not image_inputs
):
return Unpriced(usage_event_id=event.event_id, reason="usage_unknown")
pricing: Final = captured.pricing
mapped: Final = _fields(pricing.get("model_map_information"))
rates: Final = _fields(mapped.get("model_map_value"))
if (
event.call_type in {CallTypes.speech, CallTypes.aspeech}
and rates.get("input_cost_per_character") is not None
and not speech_usage
):
return Unpriced(usage_event_id=event.event_id, reason="usage_unknown")
model: Final = next(
(d.value for d in event.dimensions if d.name is DimensionName.MODEL),
event.call_type.value if fixed_call else None,
)
provider: Final = next((d.value for d in event.dimensions if d.name is DimensionName.PROVIDER), None)
if (
fixed_call
and event.outcome is not Outcome.SUCCEEDED
and captured.usage is None
and not logged.get("tool_response_cost")
):
return Unpriced(usage_event_id=event.event_id, reason="usage_unknown")
reported: Final = (
logged.get("provider_reported_cost")
if logged.get("provider_reported_cost") is not None
else logged.get("provider_usage_cost")
)
upstream: Final = (
isinstance(reported, (int, float))
and not isinstance(reported, bool)
and math.isfinite(reported)
and reported >= 0
and logged.get("custom_pricing") is not True
)
key: Final = mapped.get("model_map_key") or (model if fixed_call or upstream else None)
has_rates: Final = any(is_server_derived_pricing_key(name) and value is not None for name, value in rates.items())
if model is None or not isinstance(key, str) or (not has_rates and not fixed_call and not upstream):
return Unpriced(usage_event_id=event.event_id, reason="pricing_inputs_missing")
info: Final = {
"key": key,
"litellm_provider": provider,
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
"input_cost_per_token": None,
"output_cost_per_token": None,
"supported_openai_params": None,
**rates,
"mode": rates.get("mode") or "chat",
}
adjustments: Final = _fields(pricing.get("adjustments"))
discount: Final = adjustments.get("discount")
margin: Final = adjustments.get("margin")
secondary: Final = {
name: {
**info,
**_fields(_fields(snapshot).get("model_map_value")),
"key": _fields(snapshot).get("model_map_key") or name,
}
for name, snapshot in {**_fields(pricing.get("realtime_models")), **_fields(pricing.get("tool_models"))}.items()
if _fields(snapshot).get("model_map_value") is not None
}
base_pricing: Final = _fields(pricing.get("base_model_information"))
base_key: Final = base_pricing.get("model_map_key")
context: Final = RetainedPricing(
_json(
{
**secondary,
**(
{base_key: _fields(base_pricing.get("model_map_value"))}
if isinstance(base_key, str) and base_pricing.get("model_map_value") is not None
else {}
),
**{
alias: info
for alias in dict.fromkeys((model, key, f"{provider}/{model}", model.removeprefix(f"{provider}/")))
},
}
),
_json({provider: discount} if provider and discount is not None else {}),
_json({"global": margin} if margin is not None else {}),
)
tool_response: Final = _fields(pricing.get("tool_response"))
replay_usage: Final = Usage.model_validate(
{
**{name: value for name, value in (captured.usage or {}).items() if name != "cost"},
**({"cost": reported} if upstream and logged.get("provider_reported_cost") is None else {}),
}
)
token_response: Final = ModelResponse(
model=model,
choices=[
{
"message": {
"role": "assistant",
"annotations": tuple(
{"type": annotation}
for flag, annotation in (("web_search", "url_citation"),)
if tool_response.get(flag)
),
}
}
],
usage=replay_usage,
service_tier=response_inputs.get("service_tier"),
meta={"billed_units": response_inputs.get("billed_units")},
hidden_params={
"region_name": response_inputs.get("region_name"),
"audio_transcription_duration": response_inputs.get("audio_transcription_duration"),
},
response_ms=logged.get("response_ms"),
)
speech_response: Final = HttpxBinaryResponseContent(Response(200, content=b"")) if speech_usage else None
if speech_response is not None:
set_hidden_param(speech_response, "billed_characters", speech_characters)
speech_response.usage = replay_usage
response: Final = (
speech_response
if speech_response is not None
else LiteLLMRealtimeStreamLoggingObject.model_validate(
{
"results": pricing.get("realtime_events", []),
"usage": captured.usage or {},
"service_tier": response_inputs.get("service_tier"),
}
)
if event.call_type in {CallTypes.arealtime, CallTypes.aresponses_websocket}
else RerankResponse.model_validate(
{"id": event.event_id, "results": [], "meta": {"billed_units": response_inputs.get("billed_units")}}
)
if event.call_type in {CallTypes.rerank, CallTypes.arerank}
else ResponsesAPIResponse.model_validate(
{
"id": event.event_id,
"created_at": int(event.started_at.timestamp()),
"model": model,
"output": tool_response.get("output", []),
"usage": tool_response.get("usage"),
"service_tier": response_inputs.get("service_tier"),
"tools": tool_response.get("tools"),
"tool_usage": tool_response.get("tool_usage"),
}
)
if tool_response.get("responses_api")
else ImageResponse.model_validate(
{"created": int(event.started_at.timestamp()), "model": model, **image_inputs}
)
if image_inputs
else token_response
)
logger: Final = Logging(
model=model,
messages=[],
stream=False,
call_type=CallTypes.aresponses.value
if event.call_type in {CallTypes.get_responses, CallTypes.aget_responses}
else event.call_type.value,
start_time=event.started_at,
litellm_call_id=event.event_id,
function_id=event.attempt_id,
)
request: Final = _fields(pricing.get("request"))
deployment: Final = _fields(pricing.get("deployment"))
logger.update_environment_variables( # pyright: ignore[reportUnknownMemberType] # legacy Logging environment method accepts validated JSON fields
model=model,
custom_llm_provider=provider,
litellm_params={**deployment, **request, "metadata": {"model_info": _fields(pricing.get("model_info"))}},
additional_response_cost=request.get("additional_response_cost"),
usage=replay_usage,
end_time=event.ended_at,
mcp_tool_call_metadata={
"name": _fields(pricing.get("tool")).get("name"),
"mcp_server_cost_info": {"default_cost_per_query": _fields(pricing.get("tool")).get("cost_per_query")},
},
optional_params=dict(_fields(pricing.get("optional_params"))),
)
if logged.get("tool_response_cost") is not None:
logger.model_call_details["response_cost"] = logged["tool_response_cost"]
logger.request_pricing = context if logged.get("custom_pricing") is True else None
logger.custom_cost_per_token = TypeAdapter(CostPerToken | None).validate_python(logged.get("custom_cost_per_token"))
logger.custom_cost_per_second = TypeAdapter(float | None).validate_python(logged.get("custom_cost_per_second"))
logger.standard_built_in_tools_params = TypeAdapter(StandardBuiltInToolsParams).validate_python(
{
"web_search_options": adjustments.get("web_search_options"),
"file_search": adjustments.get("file_search") or None,
}
)
if upstream and logged.get("provider_reported_cost") is not None:
set_hidden_param(response, "additional_headers", {"llm_provider-x-litellm-response-cost": reported})
billed_at: Final = TypeAdapter(datetime).validate_python(logged.get("current_billing_time", event.ended_at))
if event.call_type is CallTypes.arealtime and captured.usage and captured.usage.get("metered_usage") is not None:
from litellm.constants import REALTIME_METERED_USAGE_KEY
from litellm.types.llms.xai import RealtimeMeteredUsage
logger.model_call_details[REALTIME_METERED_USAGE_KEY] = TypeAdapter(RealtimeMeteredUsage).validate_python(
captured.usage["metered_usage"]
)
logger.cost_billing_time = billed_at
with retained_pricing(context), pinned_billing_time(billed_at):
batch_costs: Final = (
ocr_batch_cost(
model=model,
custom_llm_provider=provider,
usage_info=OCRUsageInfo.model_validate(captured.usage),
model_info=TypeAdapter(ModelInfo).validate_python(info),
)
if event.call_type in {CallTypes.retrieve_batch, CallTypes.aretrieve_batch}
and any(name in (captured.usage or {}) for name in OCRUsageInfo.model_fields)
else batch_cost_calculator(
replay_usage,
model,
provider,
data_residency=TypeAdapter[str | None](str | None).validate_python(deployment.get("data_residency")),
vertex_location=TypeAdapter[str | None](str | None).validate_python(deployment.get("vertex_location")),
)
if event.call_type in {CallTypes.retrieve_batch, CallTypes.aretrieve_batch}
else None
)
amount: Final = (
sum(batch_costs)
if batch_costs is not None
else logger.response_cost_calculator(
response,
ignore_stamped_cost=True,
custom_pricing_override=TypeAdapter[bool | None](bool | None).validate_python(
logged.get("custom_pricing")
),
)
)
if batch_costs is not None:
logger.cost_breakdown = CostBreakdown(
input_cost=batch_costs[0], output_cost=batch_costs[1], total_cost=sum(batch_costs)
)
if amount is None:
return Unpriced(usage_event_id=event.event_id, reason="calculation_failed")
retained_final: Final = captured.finalized_cost or {}
breakdown: Final = _fields(retained_final.get("cost_breakdown"))
guardrail: Final = TypeAdapter(float).validate_python(
breakdown.get("guardrail_cost", pricing.get("guardrail_cost", 0.0))
)
allowance: Final = TypeAdapter(float).validate_python(retained_final.get("allowance_discount", 0.0))
if allowance and captured.allocation is None:
return Unpriced(usage_event_id=event.event_id, reason="allowance_allocation_missing")
finalized: Final = finalize_cost(amount + guardrail, logger.cost_breakdown, allowance)
if retained_final.get("response_cost") is not None and Decimal(str(finalized.amount)) != Decimal(
str(retained_final["response_cost"])
):
return Unpriced(usage_event_id=event.event_id, reason="finalized_cost_mismatch")
final_breakdown: Final = dict(finalized.breakdown or {})
components: Final = {
name: Decimal(str(value))
for name, value in final_breakdown.items()
if name in _COST_COMPONENTS and isinstance(value, (int, float))
}
return PricingSnapshot.model_validate(
{
**components,
"calculator_version": "logging-v1",
"model": model,
"provider": provider,
"pricing_source": "upstream_header"
if upstream
else "per_request"
if any(is_server_derived_pricing_key(name) and value is not None for name, value in request.items())
else "deployment"
if any(is_server_derived_pricing_key(name) and value is not None for name, value in deployment.items())
else "cost_map",
"priced_model_key": key,
"total_cost": Decimal(str(finalized.amount)),
"guardrail_cost": Decimal(str(guardrail)),
"upstream_reported_cost": Decimal(str(reported)) if upstream else None,
"additional_costs": tuple(
{"name": name, "amount": Decimal(str(value))}
for name, value in {
**_fields(final_breakdown.get("additional_costs")),
**(
{"request": request["additional_response_cost"]}
if request.get("additional_response_cost")
else {}
),
}.items()
),
}
)
_COST_COMPONENTS: Final = frozenset(
{
"input_cost",
"output_cost",
"cache_read_cost",
"cache_creation_cost",
"reasoning_cost",
"tool_usage_cost",
"original_cost",
"discount_percent",
"discount_amount",
"margin_percent",
"margin_fixed_amount",
"margin_total_amount",
}
)
class LoggingCostCompiler:
def __init__(
self,
ledger: AccountingLedger,
price_usage: Callable[[str], PricingSnapshot | Unpriced] = price_usage,
delivery_targets: tuple[DeliveryTarget, ...] | None = None,
) -> None:
self._ledger: Final = ledger
self._price_usage: Final = price_usage
self._delivery_targets: Final = delivery_targets
async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
targets: Final = self._delivery_targets if self._delivery_targets is not None else cost_delivery_targets(event)
return await self._ledger.compile_cost(event, self._price_usage, targets)
async def compile_pending(self, limit: int = 100) -> None:
for event in await self._ledger.pending_compilation(limit):
try:
await self.compile(event)
except Exception: # noqa: BLE001 # retain pending work for retry without starving other events
verbose_proxy_logger.exception("Cost compilation failed for %s", event.event_id)
try:
await self._ledger.defer_compilation(event)
except Exception: # noqa: BLE001 # database outages must not prevent attempts on later events
verbose_proxy_logger.exception(
"Cost compilation retry could not be scheduled for %s", event.event_id
)
_SPEND_TARGETS: Final = (
(DimensionName.KEY, DeliveryTarget.KEY_SPEND),
(DimensionName.USER, DeliveryTarget.USER_SPEND),
(DimensionName.TEAM, DeliveryTarget.TEAM_SPEND),
(DimensionName.ORGANIZATION, DeliveryTarget.ORGANIZATION_SPEND),
(DimensionName.END_USER, DeliveryTarget.END_USER_SPEND),
(DimensionName.AGENT, DeliveryTarget.AGENT_SPEND),
(DimensionName.TAG, DeliveryTarget.TAG_SPEND),
(DimensionName.PROJECT, DeliveryTarget.PROJECT_SPEND),
(DimensionName.MODEL_ACCESS_GROUP, DeliveryTarget.MODEL_ACCESS_GROUP_SPEND),
)
def cost_delivery_targets(event: OutboundUsageEvent) -> tuple[DeliveryTarget, ...]:
names: Final = frozenset(d.name for d in event.dimensions)
return (
*usage_delivery_targets(event.dimensions),
*(target for dimension, target in _SPEND_TARGETS if dimension in names),
*((DeliveryTarget.TEAM_MEMBER_SPEND,) if {DimensionName.TEAM, DimensionName.USER} <= names else ()),
*(
(DeliveryTarget.ORGANIZATION_MEMBER_SPEND,)
if {DimensionName.ORGANIZATION, DimensionName.USER} <= names
else ()
),
)

View file

@ -8,13 +8,14 @@ from dataclasses import dataclass, replace
from datetime import datetime, timezone
from decimal import Decimal
from hashlib import sha256
from itertools import chain, groupby, islice, repeat
from itertools import chain, groupby, islice, product, repeat
from typing import TYPE_CHECKING, Final
from pydantic import BaseModel, JsonValue, TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.accounting import (
CostCompiler,
Dimension,
DimensionName,
InboundUsageEvent,
@ -30,7 +31,7 @@ from litellm.proxy.spend_tracking.usage_meter import (
UsageCaptureSettings,
usage_delivery_targets,
)
from litellm.types.utils import CallTypes, CustomPricingLiteLLMParams
from litellm.types.utils import CallTypes, CustomPricingLiteLLMParams, ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
@ -172,7 +173,15 @@ def usage_writes(
ended_at=ended,
outcome=outcome,
dimensions=dimensions,
metrics=_usage_metrics(None if context.usage_source == "unknown" else usage),
metrics=_usage_metrics(
None
if context.usage_source == "unknown"
or (
context.ownership == "aggregate"
and payload["call_type"] in {CallTypes.retrieve_batch.value, CallTypes.aretrieve_batch.value}
)
else usage
),
)
envelope: Final = {
"event": outbound.model_dump(mode="json"),
@ -217,6 +226,7 @@ def usage_writes(
_INCLUDED_IN_PARENT: Final = object()
_BATCH_ROWS: Final = object()
_PRICING_FIELDS: Final = frozenset(
@ -240,6 +250,7 @@ _PRICING_FIELDS: Final = frozenset(
"seconds",
"search_context_size",
"additional_response_cost",
"cost_per_query",
}
)
@ -279,6 +290,7 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res
mask_api_base_credentials,
)
from litellm.litellm_core_utils.llm_cost_calc.finalization import FinalizedCost
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation
from litellm.proxy.spend_tracking.spend_tracking_utils import get_request_model_access_groups
@ -303,9 +315,14 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res
"expired",
"incomplete",
}
batch_summary: Final = (
payload["call_type"] in {CallTypes.retrieve_batch.value, CallTypes.aretrieve_batch.value}
and kwargs.get("_accounting_batch_rows") is not _BATCH_ROWS
)
deferred: Final = billed_id.endswith(("_batch_cost", "_background_cost")) and terminal
management: Final = (
(billed_id.endswith(("_batch_cost", "_background_cost")) and not terminal)
payload["call_type"] in {CallTypes.create_batch.value, CallTypes.acreate_batch.value}
or (billed_id.endswith(("_batch_cost", "_background_cost")) and not terminal)
or (is_background_response(response) and not terminal)
or is_unbilled_non_inference_call(payload["call_type"], request_metadata, response)
)
@ -316,6 +333,8 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res
if management
else f"{billed_id}:included"
if included_in_parent
else f"{billed_id}:summary"
if batch_summary and terminal
else billed_id
)
if not attempt or attempt == "None" or (cached and not caller):
@ -333,7 +352,11 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res
else kwargs.get("_accounting_allocation")
)
allocation: Final = allocation_value if isinstance(allocation_value, SearchAllowanceAllocation) else None
pricing_snapshot: Final = kwargs.get("_accounting_pricing_snapshot")
pricing_snapshot: Final = (
_capture_pricing(logging_obj, response, kwargs)
if isinstance(logging_obj, Logging) and _batch_rows(response)
else kwargs.get("_accounting_pricing_snapshot")
)
observation: Final = kwargs.get("_accounting_usage_snapshot")
captured_pricing: Final = {
"model_map_information": (
@ -342,6 +365,7 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res
else metadata.get("model_map_information")
),
"cost_breakdown": metadata.get("cost_breakdown"),
"guardrail_cost": guardrail_information_cost(metadata.get("guardrail_information")),
**(
_JSON_OBJECT.validate_json(pricing_snapshot.inputs)
if isinstance(pricing_snapshot, PricingCapture)
@ -381,12 +405,12 @@ def capture_context(payload: SpendLogsPayload, kwargs: Mapping[str, object], res
parent_request_id=parent,
retry_of_attempt_id=_text(request_metadata.get("retry_of_attempt_id")),
job_id=_text(raw_response.get("id")) if deferred or management else None,
charge_eligible=not cached and not included_in_parent and not management,
charge_eligible=not cached and not included_in_parent and not management and not batch_summary,
request_count_eligible=bool(caller) and not origin and not retry and not deferred and not included_in_parent,
ownership="included_in_parent"
if included_in_parent
else "aggregate"
if kwargs.get("additional_response_cost") is not None
if batch_summary or kwargs.get("additional_response_cost") is not None
else "attempt",
pricing=_json(captured_pricing),
finalized_cost=_json(
@ -555,13 +579,26 @@ def _batch_rows(response: object) -> tuple[Mapping[str, JsonValue], ...]:
def _batch_root(write: UsageWrite, rows: tuple[Mapping[str, JsonValue], ...]) -> UsageWrite:
event: Final = write.event.model_copy(update={"metrics": None})
from litellm.proxy.openai_files_endpoints.common_utils import (
convert_b64_uid_to_unified_uid,
get_batch_id_from_unified_batch_id,
get_original_file_id,
)
envelope: Final = _JSON_OBJECT.validate_json(write.envelope)
observed_job: Final = TypeAdapter(str).validate_python(envelope["job_id"])
decoded: Final = convert_b64_uid_to_unified_uid(observed_job)
job: Final = (
get_batch_id_from_unified_batch_id(decoded) if decoded != observed_job else get_original_file_id(observed_job)
)
identity: Final = f"{job}_batch_cost"
event: Final = write.event.model_copy(update={"metrics": None, "event_id": identity, "attempt_id": identity})
return _database_usage_write(
event,
{
**envelope,
"event": event.model_dump(mode="json"),
"job_id": job,
"usage": {
"batch_row_count": len(rows),
"batch_rows_sha256": sha256(_json(rows).encode()).hexdigest(),
@ -635,7 +672,7 @@ def _prepare_batch_capture(
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
rows: Final = tuple(sorted(_batch_rows(response), key=_json))
builder: Final = _CaptureBuilder(kwargs, response)
builder: Final = _CaptureBuilder({**kwargs, "_accounting_batch_rows": _BATCH_ROWS}, response)
payload: Final = get_logging_payload(
dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe
)
@ -782,10 +819,27 @@ async def capture_proxy_response(logging_obj: object, response: object, *, backg
timeout: Final = TypeAdapter(float).validate_python(
settings.get("accounting_sync_ack_timeout_seconds", 20.0)
)
await asyncio.wait_for(asyncio.shield(captured.receipt), timeout=timeout)
async with asyncio.timeout(timeout):
await asyncio.shield(captured.receipt)
compiler: Final = usage_capture.compiler
if compiler is not None:
for write in captured.writes:
if isinstance(write.event, OutboundUsageEvent):
await compiler.compile(write.event)
except Exception as error: # noqa: BLE001 # the configured response policy owns every capture failure
verbose_proxy_logger.exception("Usage capture failed before the response boundary")
if sync_ack and settings.get("accounting_sync_ack_failure_policy", "fail_closed") == "fail_closed":
accepted: Final = logging_obj.model_call_details.get("_accounting_capture")
committed: Final = (
isinstance(accepted, CapturedUsage)
and accepted.receipt.done()
and not accepted.receipt.cancelled()
and accepted.receipt.exception() is None
)
if (
sync_ack
and not committed
and settings.get("accounting_sync_ack_failure_policy", "fail_closed") == "fail_closed"
):
acknowledgement: Final = UsageAcknowledgementError(error)
raise HTTPException(503, str(acknowledgement)) from acknowledgement
@ -828,10 +882,10 @@ def capture_logging_usage(
existing_pricing: Final = details.get("_accounting_pricing_snapshot")
pricing: Final = (
existing_pricing
if isinstance(existing_pricing, PricingCapture) and not internal
else _capture_pricing(logging_obj, normalized, details)
if isinstance(existing_pricing, PricingCapture) and not internal and not _batch_rows(normalized)
else _capture_pricing(logging_obj, normalized_result, details)
)
if freeze_only or _batch_rows(normalized):
if freeze_only or logging_obj.call_type in {CallTypes.retrieve_batch.value, CallTypes.aretrieve_batch.value}:
if not internal:
logging_obj.record_usage_pricing(pricing)
return
@ -866,9 +920,21 @@ class PricingCapture:
def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[str, object]) -> PricingCapture:
from litellm.litellm_core_utils.llm_cost_calc.pricing_context import retained_pricing
if logging_obj.request_pricing is not None and not logging_obj.spilled_over_ptu_for(normalized):
with retained_pricing(logging_obj.request_pricing):
return _capture_effective_pricing(logging_obj, normalized, details)
return _capture_effective_pricing(logging_obj, normalized, details)
def _capture_effective_pricing(
logging_obj: Logging, normalized: object, details: Mapping[str, object]
) -> PricingCapture:
import litellm
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.utils import get_base_model_from_metadata
params: Final = _fields(details.get("litellm_params"))
prepared: Final = logging_obj.prepared_cost
@ -879,7 +945,7 @@ def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[
if prepared is not None
else StandardLoggingPayloadSetup.get_model_cost_information(
base_model=logging_obj.get_deployment_model_for_cost(),
custom_pricing=bool(params.get("custom_pricing")),
custom_pricing=logging_obj.custom_pricing_for(normalized),
custom_llm_provider=_text(details.get("custom_llm_provider")),
init_response_obj=normalized,
api_base=_text(params.get("api_base")),
@ -900,16 +966,62 @@ def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[
if builtins.get("web_search_options") is not None
else None
),
"file_search": builtins.get("file_search") is not None,
"file_search": {
key: value
for key, value in _fields(builtins.get("file_search")).items()
if key in {"type", "storage_gb", "days"}
}
if builtins.get("file_search") is not None
else None,
}
)
batch_usage: Final = _fields(_fields(_response_fields(normalized).get("usage")).get("batch_usage_by_model"))
base_model: Final = get_base_model_from_metadata(model_call_details=logging_obj.model_call_details)
realtime_models: Final = tuple(
dict.fromkeys(
chain.from_iterable(_realtime_pricing_models(event) for event in _realtime_cost_events(normalized))
)
)
batch_usage: Final = _fields(
_response_fields(_response_fields(normalized).get("usage")).get("batch_usage_by_model")
)
return PricingCapture(
_json(pricing),
adjustments,
_json(
{
**_pricing_inputs(details, logging_obj, normalized),
"base_model_information": StandardLoggingPayloadSetup.get_model_cost_information(
base_model=base_model,
custom_pricing=False,
custom_llm_provider=None,
init_response_obj={"model": base_model},
)
if base_model is not None
else None,
"realtime_models": {
model: StandardLoggingPayloadSetup.get_model_cost_information(
base_model=model,
custom_pricing=False,
custom_llm_provider=provider,
init_response_obj={"model": model},
)
for model in realtime_models
},
"tool_models": {
model: StandardLoggingPayloadSetup.get_model_cost_information(
base_model=model,
custom_pricing=False,
custom_llm_provider=provider,
init_response_obj={"model": model},
)
for model in _image_pricing_models(
normalized,
provider,
logging_obj.get_deployment_model_for_cost(),
_fields(details.get("optional_params")),
)
if model in _fields(vars(litellm)["model_cost"])
},
"batch_models": {
model: StandardLoggingPayloadSetup.get_model_cost_information(
base_model=model,
@ -926,10 +1038,168 @@ def _capture_pricing(logging_obj: Logging, normalized: object, details: Mapping[
)
def _tool_cost_inputs(response: object) -> Mapping[str, object]:
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import StandardBuiltInToolCostTracking
from litellm.types.llms.openai import ResponsesAPIResponse
raw: Final = _response_fields(response)
flags: Final = {
"web_search": StandardBuiltInToolCostTracking.response_object_includes_web_search_call(response),
"file_search": StandardBuiltInToolCostTracking.response_object_includes_file_search_call(response),
}
if not isinstance(response, ResponsesAPIResponse) and raw.get("object") != "response":
return flags if any(flags.values()) else {}
return {
**flags,
"responses_api": True,
"tools": tuple(
{key: value for key, value in _fields(tool).items() if key in {"type", "model"}}
for tool in TypeAdapter(tuple[object, ...]).validate_python(raw.get("tools") or ())
if _fields(tool).get("type") == "image_generation"
),
"tool_usage": _response_fields(raw.get("tool_usage")) or None,
"usage": _response_fields(raw.get("usage")),
"output": tuple(
{
key: value
for key, value in _fields(item).items()
if key in {"type", "id", "status", "usage", "quality", "size"}
}
for item in TypeAdapter(tuple[object, ...]).validate_python(raw.get("output") or ())
),
}
def _image_pricing_models(
response: object, provider: str | None, deployment_model: str | None, optional_params: Mapping[str, object]
) -> tuple[str, ...]:
from litellm.cost_calculator import image_cost_model_candidates
from litellm.litellm_core_utils.llm_cost_calc.utils import image_response_size_and_quality
if isinstance(response, ImageResponse) and deployment_model is not None:
size, quality = image_response_size_and_quality(response, optional_params)
return tuple(
key
for key in image_cost_model_candidates(
deployment_model,
provider,
quality,
size,
)
if key is not None
)
raw: Final = _tool_cost_inputs(response)
models: Final = tuple(
model
for tool in TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(raw.get("tools") or ())
if isinstance(model := tool.get("model"), str)
)
items: Final = tuple(
item
for item in TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(raw.get("output") or ())
if item.get("type") == "image_generation_call" and item.get("status") == "completed"
)
candidates: Final = chain.from_iterable(
image_cost_model_candidates(
model,
provider,
_text(item.get("quality")) if item.get("quality") != "auto" else None,
_text(item.get("size")) if item.get("size") != "auto" else None,
)
for model, item in product(models, items)
)
return tuple(dict.fromkeys(model for model in candidates if model is not None))
_REALTIME_CHILDREN: Final = (
"response",
"event",
"session",
"audio",
"input",
"transcription",
"input_audio_transcription",
)
def _realtime_cost_node(
fields: Mapping[str, object],
) -> dict[str, JsonValue]: # mutable-ok: tree builder attaches sanitized children
return {
**{
key: _JSON_OBJECT.validate_python(fields[key]) for key in ("usage",) if isinstance(fields.get(key), Mapping)
},
**{
key: value
for key, value in fields.items()
if key in {"type", "id", "model", "service_tier"} and isinstance(value, str)
},
}
def _realtime_cost_fields(value: object) -> Mapping[str, JsonValue]:
root: Final = _realtime_cost_node(_fields(value))
pending: Final[list[tuple[Mapping[str, object], dict[str, JsonValue]]]] = [ # mutable-ok: traversal stack
(_fields(value), root)
]
while pending:
source, target = pending.pop()
for key in _REALTIME_CHILDREN:
if not isinstance(source.get(key), Mapping):
continue
fields: Final = _fields(source[key]) # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value
child: Final = _realtime_cost_node(fields) # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value
target[key] = child
pending.append((fields, child))
return root
def _realtime_cost_events(response: object) -> tuple[Mapping[str, JsonValue], ...]:
results: Final = _response_fields(response).get("results")
return (
tuple(_realtime_cost_fields(result) for result in TypeAdapter(tuple[object, ...]).validate_python(results))
if isinstance(results, list)
else ()
)
def _realtime_pricing_models(value: Mapping[str, JsonValue]) -> Iterator[str]:
pending: Final[list[Mapping[str, JsonValue]]] = [ # mutable-ok: iterative traversal avoids recursion
value
]
while pending:
fields: Final = pending.pop() # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value
model: Final = fields.get("model") # pyright: ignore[reportGeneralTypeIssues] # loop-local traversal value
if isinstance(model, str):
yield model
pending.extend(nested for key in _REALTIME_CHILDREN if isinstance(nested := fields.get(key), dict))
def _image_cost_dimensions(image: object) -> Mapping[str, object]:
fields: Final = _fields(_response_fields(image).get("provider_specific_fields"))
return {
"provider_specific_fields": {
name: value
for name, value in fields.items()
if name in {"width", "height"} and isinstance(value, int) and not isinstance(value, bool)
}
}
def _pricing_inputs(kwargs: Mapping[str, object], logging_obj: object, response: object) -> Mapping[str, object]:
from litellm.litellm_core_utils.litellm_logging import Logging
return {
"image_response": {
"data": tuple(_image_cost_dimensions(image) for image in response.data or ()),
"usage": _response_fields(response.usage) if response.usage is not None else None,
"size": response.size,
"quality": response.quality,
}
if isinstance(response, ImageResponse)
else None,
"tool_response": _tool_cost_inputs(response),
"realtime_events": _realtime_cost_events(response),
"request": _selected_pricing(kwargs),
"deployment": _selected_pricing(_fields(kwargs.get("litellm_params"))),
"optional_params": _selected_pricing(_fields(kwargs.get("optional_params"))),
@ -1006,6 +1276,7 @@ class UsageCaptureRuntime:
self._batches: Final[set[asyncio.Task[None]]] = set() # mutable-ok: owned producer lifecycle
self._stopping = False
self.meter: DurableUsageMeter | None = None
self.compiler: CostCompiler | None = None
self.load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None
def start(
@ -1014,6 +1285,7 @@ class UsageCaptureRuntime:
settings: UsageCaptureSettings,
*,
load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None,
compiler: CostCompiler | None = None,
) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
@ -1024,6 +1296,7 @@ class UsageCaptureRuntime:
meter: Final = DurableUsageMeter(write_batch, settings)
self.meter = meter
self.load = load
self.compiler = compiler
self._stopping = False
Logging.usage_capture_callback = lambda logger, response, start, end, freeze: capture_logging_usage(
meter, logger, response, start, end, freeze
@ -1082,3 +1355,4 @@ class UsageCaptureRuntime:
Logging.usage_capture_callback = None
self.meter = None
self.load = None
self.compiler = None

View file

@ -437,6 +437,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
output_cost_per_token_above_512k_tokens: float | None # MiniMax-M3: prompts >512K priced at 2x output
output_cost_per_character_above_128k_tokens: float | None # only for vertex ai models
output_cost_per_image: float | None
input_cost_per_pixel: ReadOnly[float | None]
output_cost_per_pixel: ReadOnly[float | None]
output_cost_per_image_token: float | None
output_cost_per_image_token_batches: ReadOnly[float | None]
@ -488,6 +489,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
"extraction",
"ocr",
"realtime",
"rerank",
]
]
supported_endpoints: list[str] | None

View file

@ -7327,6 +7327,7 @@ def get_model_info_helper(
output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None),
output_cost_per_video_per_second=_model_info.get("output_cost_per_video_per_second", None),
output_cost_per_image=_model_info.get("output_cost_per_image", None),
input_cost_per_pixel=_model_info.get("input_cost_per_pixel", None),
output_cost_per_pixel=_model_info.get("output_cost_per_pixel", None),
output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None),
output_cost_per_image_token_batches=_model_info.get("output_cost_per_image_token_batches", None),
@ -7531,6 +7532,11 @@ def get_model_info(
"supported_openai_params": ["temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty"]
}
"""
from litellm.litellm_core_utils.llm_cost_calc.pricing_context import retained_model_info
retained: Final = retained_model_info(model, custom_llm_provider)
if retained is not None:
return retained
# api_key is a per-caller credential, not part of the model identity, so it is
# kept out of the cache key; explicit keys are resolved without the cache.
if api_key is not None:

View file

@ -2111,8 +2111,12 @@ model LiteLLM_AccountingEvent {
occurred_at DateTime
recorded_at DateTime @default(now())
payload Json
compilation_state String @default("pending")
compilation_error String?
compilation_retry_at DateTime?
@@index([recorded_at])
@@index([kind, compilation_state, recorded_at])
@@index([kind, source_event_id])
}

View file

@ -20,6 +20,7 @@ from psycopg import sql
from pydantic import JsonValue
from litellm.accounting import (
CostCompiler,
CostEvent,
DeliveryState,
DeliveryTarget,
@ -30,9 +31,11 @@ from litellm.accounting import (
OutboundUsageEvent,
Outcome,
PricingSnapshot,
Unpriced,
)
from litellm.proxy.db import accounting_ledger
from litellm.proxy.db.accounting_ledger import AccountingConflictError, AccountingEvent, AccountingLedger, UsageWrite
from litellm.proxy.spend_tracking.cost_compiler import LoggingCostCompiler, price_usage
from litellm.types.utils import CallTypes
from tests.integration._support.database import scratch_database
from tests.integration._support.prisma_clients import connected_proxy_prisma
@ -248,6 +251,184 @@ async def test_event_and_complete_delivery_set_commit_together(writer: _Writer,
assert set(deliveries) == {(target.value, "pending", 0) for target in _TARGETS}
@pytest.mark.asyncio
async def test_compilers_share_one_committed_charge_and_preserve_delivery_receipts(
writer: _Writer, ledger_database: str
) -> None:
_, store = writer
event: Final = _usage()
assert isinstance(event, OutboundUsageEvent)
await _insert(store, event, context={"price": "0.125"})
def price(envelope: str) -> PricingSnapshot:
return _cost(amount=json.loads(envelope)["context"]["price"]).snapshot
compiler: Final[CostCompiler] = LoggingCostCompiler(store, price, _TARGETS)
results: Final = await asyncio.gather(*(compiler.compile(event) for _ in range(8)))
assert all(result == results[0] for result in results)
assert isinstance(results[0], CostEvent)
assert results[0].snapshot.total_cost == Decimal("0.125")
assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(1,)]
deliveries: Final = _rows(
ledger_database,
'SELECT target::text, state::text FROM "LiteLLM_AccountingDelivery" WHERE event_id = %s',
(results[0].event_id,),
)
assert set(deliveries) == {(target.value, "pending") for target in _TARGETS}
_complete_and_age(ledger_database)
before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target')
def unavailable_pricing(envelope: str) -> PricingSnapshot:
raise AssertionError("Committed cost replay must not invoke pricing")
replay: Final = await LoggingCostCompiler(store, unavailable_pricing, _TARGETS).compile(event)
assert replay == results[0]
assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before
@pytest.mark.asyncio
async def test_compilation_rejects_changed_delivery_targets_without_resetting_receipts(
writer: _Writer, ledger_database: str
) -> None:
_, store = writer
event: Final = _usage()
assert isinstance(event, OutboundUsageEvent)
await _insert(store, event)
original: Final = await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS[:1]).compile(event)
_complete_and_age(ledger_database)
before: Final = _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target')
with pytest.raises(AccountingConflictError, match="delivery set"):
await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event)
assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before
assert await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS[:1]).compile(event) == original
@pytest.mark.asyncio
async def test_compilation_preserves_unknown_cost_without_creating_a_zero_charge(
writer: _Writer, ledger_database: str
) -> None:
_, store = writer
event: Final = _usage()
assert isinstance(event, OutboundUsageEvent)
unresolved: Final = Unpriced(usage_event_id=event.event_id, reason="unknown_usage")
compiler: Final = LoggingCostCompiler(store, lambda _: unresolved, _TARGETS)
assert await compiler.compile(event) == Unpriced(usage_event_id=event.event_id, reason="usage_not_committed")
await _insert(store, event)
assert await compiler.compile(event) == unresolved
assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)]
@pytest.mark.asyncio
async def test_compilation_rejects_an_unpriced_result_for_another_source(writer: _Writer, ledger_database: str) -> None:
_, store = writer
event: Final = _usage()
assert isinstance(event, OutboundUsageEvent)
await _insert(store, event)
compiler: Final = LoggingCostCompiler(
store, lambda _: Unpriced(usage_event_id="other", reason="missing_price"), _TARGETS
)
with pytest.raises(AccountingConflictError, match="Unpriced result"):
await compiler.compile(event)
assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)]
@pytest.mark.asyncio
async def test_compilation_rejects_a_cost_identity_collision(writer: _Writer, ledger_database: str) -> None:
_, store = writer
event: Final = _usage()
assert isinstance(event, OutboundUsageEvent)
await _insert(store, event)
await _insert(store, _usage(f"cost:{event.event_id}", inbound=True))
with pytest.raises(AccountingConflictError, match="identity_conflict"):
await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event)
assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)]
@pytest.mark.asyncio
async def test_compilation_rejects_changed_source_before_reusing_a_charge(writer: _Writer) -> None:
_, store = writer
event: Final = _usage()
assert isinstance(event, OutboundUsageEvent)
await _insert(store, event)
compiler: Final = LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS)
assert isinstance(await compiler.compile(event), CostEvent)
changed: Final = event.model_copy(update={"outcome": Outcome.CANCELLED})
with pytest.raises(AccountingConflictError, match="differs from the accepted usage"):
await compiler.compile(changed)
@pytest.mark.asyncio
async def test_compilation_failure_retains_retry_without_a_partial_charge(
writer: _Writer, ledger_database: str
) -> None:
_, store = writer
event: Final = _usage()
assert isinstance(event, OutboundUsageEvent)
await _insert(store, event)
def failed_pricing(envelope: str) -> PricingSnapshot:
raise RuntimeError("pricing temporarily unavailable")
assert await LoggingCostCompiler(store, failed_pricing, _TARGETS).compile(event) == Unpriced(
usage_event_id=event.event_id, reason="calculation_retryable"
)
assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)]
result: Final = await LoggingCostCompiler(store, lambda _: _cost(amount="0").snapshot, _TARGETS).compile(event)
assert isinstance(result, CostEvent)
assert result.charges[0].value == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("kind", ["usage", "cost"])
async def test_compilation_does_not_interpret_unknown_envelope_versions(writer: _Writer, kind: str) -> None:
_, store = writer
event: Final = _usage()
assert isinstance(event, OutboundUsageEvent)
await store.insert_event(event, _TARGETS, envelope=_envelope(event), schema_version=2 if kind == "usage" else 1)
if kind == "cost":
cost: Final = _cost()
await store.insert_event(cost, _TARGETS, envelope=_envelope(cost), schema_version=2)
result: Final = await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event)
assert result == Unpriced(usage_event_id=event.event_id, reason=f"unsupported_{kind}_schema")
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["before_commit", "after_commit", "cancellation"])
async def test_compiler_recovery_preserves_atomic_charge_and_delivery_acceptance(
writer: _Writer, ledger_database: str, failure: str
) -> None:
client, store = writer
event: Final = _usage()
assert isinstance(event, OutboundUsageEvent)
await _insert(store, event)
@asynccontextmanager
async def interrupted() -> AsyncGenerator[Prisma]:
async with client.writer_db.tx() as tx:
yield tx
if failure == "before_commit":
raise RuntimeError("interrupted cost commit")
if failure == "cancellation":
raise asyncio.CancelledError
raise RuntimeError("lost cost acknowledgement")
compiler: Final = LoggingCostCompiler(AccountingLedger(interrupted), lambda _: _cost().snapshot, _TARGETS)
error: Final = asyncio.CancelledError if failure == "cancellation" else RuntimeError
with pytest.raises(error):
await compiler.compile(event)
count: Final = int(failure == "after_commit")
assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(count,)]
assert _rows(
ledger_database,
'SELECT count(*) FROM "LiteLLM_AccountingDelivery" WHERE event_id = %s',
(f"cost:{event.event_id}",),
) == [(count * len(_TARGETS),)]
result: Final = await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(event)
assert isinstance(result, CostEvent)
assert result.snapshot == _cost().snapshot
assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(1,)]
@pytest.mark.asyncio
async def test_concurrent_duplicates_preserve_completed_deliveries(writer: _Writer, ledger_database: str) -> None:
_, store = writer
@ -608,3 +789,345 @@ async def test_invalid_jsonb_submission_cannot_reject_another_producers_commit(
assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("valid",)]
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)]
assert (meter.pending_bytes, meter.pending_records) == (0, 0)
@pytest.mark.asyncio
async def test_compilation_persists_invalid_pricing_reason_without_charging(
writer: _Writer, ledger_database: str
) -> None:
_, store = writer
event: Final = _usage()
assert isinstance(event, OutboundUsageEvent)
await _insert(store, event)
accepted: Final = _rows(
ledger_database,
'SELECT payload, payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = %s',
(event.event_id,),
)
def malformed(envelope: str) -> PricingSnapshot:
raise ValueError("invalid private input must not appear in durable diagnostics")
result: Final = await LoggingCostCompiler(store, malformed, _TARGETS).compile(event)
assert result == Unpriced(usage_event_id=event.event_id, reason="invalid_pricing_inputs")
assert _rows(
ledger_database,
'SELECT compilation_state, compilation_error FROM "LiteLLM_AccountingEvent" WHERE event_id = %s',
(event.event_id,),
) == [("unpriced", "invalid_pricing_inputs")]
assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(0,)]
assert (
_rows(
ledger_database,
'SELECT payload, payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = %s',
(event.event_id,),
)
== accepted
)
async with connected_proxy_prisma(ledger_database) as restarted:
assert await AccountingLedger.for_client(restarted).pending_compilation() == ()
@pytest.mark.asyncio
async def test_pending_compilation_only_returns_unresolved_billable_work(writer: _Writer, ledger_database: str) -> None:
_, store = writer
events: Final = tuple(_usage(name) for name in ("compiled", "excluded", "unknown", "pending"))
assert all(isinstance(event, OutboundUsageEvent) for event in events)
for event in events:
await _insert(store, event)
await _insert(store, _usage("inbound", inbound=True))
compiled, excluded, unknown, pending = events
assert isinstance(compiled, OutboundUsageEvent)
assert isinstance(excluded, OutboundUsageEvent)
assert isinstance(unknown, OutboundUsageEvent)
assert isinstance(pending, OutboundUsageEvent)
assert isinstance(
await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(compiled), CostEvent
)
await LoggingCostCompiler(
store, lambda _: Unpriced(usage_event_id=excluded.event_id, reason="not_charge_owner"), _TARGETS
).compile(excluded)
await LoggingCostCompiler(
store, lambda _: Unpriced(usage_event_id=unknown.event_id, reason="usage_unknown"), _TARGETS
).compile(unknown)
assert await store.pending_compilation(limit=1) == (pending,)
assert _rows(
ledger_database,
"SELECT event_id, compilation_state FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'outbound_usage' ORDER BY event_id",
) == [("compiled", "compiled"), ("excluded", "excluded"), ("pending", "pending"), ("unknown", "unpriced")]
@pytest.mark.asyncio
async def test_restarted_compiler_recovers_transient_failures_without_revisiting_terminal_work(
writer: _Writer, ledger_database: str
) -> None:
_, store = writer
delayed: Final = _usage("delayed")
healthy: Final = _usage("healthy")
assert isinstance(delayed, OutboundUsageEvent)
assert isinstance(healthy, OutboundUsageEvent)
await _insert(store, delayed)
await _insert(store, healthy)
def transient(envelope: str) -> PricingSnapshot:
if OutboundUsageEvent.model_validate(json.loads(envelope)["event"]).event_id == delayed.event_id:
raise RuntimeError("temporary pricing dependency failure")
return _cost().snapshot
await LoggingCostCompiler(store, transient, _TARGETS).compile_pending()
assert await store.pending_compilation() == ()
assert _rows(
ledger_database,
"SELECT compilation_state, compilation_error FROM \"LiteLLM_AccountingEvent\" WHERE event_id = 'delayed'",
) == [("pending", "calculation_retryable")]
assert _rows(ledger_database, "SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [
(healthy.event_id,)
]
async with connected_proxy_prisma(ledger_database) as restarted:
recovered: Final = AccountingLedger.for_client(restarted)
await restarted.db.execute_raw(
'UPDATE "LiteLLM_AccountingEvent" SET compilation_retry_at = NULL WHERE event_id = $1', delayed.event_id
)
await LoggingCostCompiler(recovered, lambda _: _cost().snapshot, _TARGETS).compile_pending()
assert await recovered.pending_compilation() == ()
await LoggingCostCompiler(recovered, transient, _TARGETS).compile_pending()
assert _rows(
ledger_database,
"SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost' ORDER BY source_event_id",
) == [(delayed.event_id,), (healthy.event_id,)]
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_ack", (False, True))
@pytest.mark.parametrize("provider_header", (False, True))
async def test_proxy_capture_and_background_compiler_converge_on_final_header_charge(
writer: _Writer, ledger_database: str, monkeypatch: pytest.MonkeyPatch, sync_ack: bool, provider_header: bool
) -> None:
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy import proxy_server
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
from litellm.proxy.spend_tracking.cost_finalization import finalize_response_cost
from litellm.proxy.spend_tracking.usage_capture import UsageBatchWriter, UsageCaptureRuntime, capture_proxy_response
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from litellm.types.utils import ModelResponse, Usage
_, store = writer
entered: Final = asyncio.Event()
release: Final = asyncio.Event()
batches: Final = UsageBatchWriter(store.insert_usage_batch, store.load_usage_batch)
async def delayed_write(writes: tuple[UsageWrite, ...]) -> None:
entered.set()
await release.wait()
await batches.write(writes)
compiler: Final = LoggingCostCompiler(store)
runtime: Final = UsageCaptureRuntime()
runtime.start(delayed_write, UsageCaptureSettings(batch_size=2), load=store.load_usage_batch, compiler=compiler)
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": sync_ack})
logger: Final = Logging(
model="capture-priced-model",
messages=[],
stream=False,
call_type="acompletion",
start_time=_AT,
litellm_call_id="gateway-request",
function_id="gateway-attempt",
)
logger.update_environment_variables(
model="capture-priced-model", custom_llm_provider="openai", litellm_params={}, optional_params={}
)
if not provider_header:
logger.set_request_pricing(
"capture-priced-model", "openai", {"input_cost_per_token": 0.01, "output_cost_per_token": 0.02}
)
response: Final = ModelResponse(
id="gateway-provider", model="capture-priced-model", usage=Usage(prompt_tokens=10, completion_tokens=4)
)
if provider_header:
from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params
set_response_cost_in_hidden_params(response, 0.18)
try:
finalized: Final = await finalize_response_cost(logger, response, None, _AT, guardrail_cost=0.03)
assert finalized is not None
headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers(
user_api_key_dict=UserAPIKeyAuth(), response_cost=finalized.amount, litellm_logging_obj=logger
)
capture: Final = asyncio.create_task(capture_proxy_response(logger, response))
await asyncio.wait_for(entered.wait(), timeout=5)
if sync_ack:
assert not capture.done()
else:
await asyncio.wait_for(asyncio.shield(capture), timeout=5)
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(0,)]
release.set()
await asyncio.wait_for(capture, timeout=10)
assert runtime.meter is not None
await runtime.meter.flush()
assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [
(1 if sync_ack else 0,)
]
await asyncio.gather(*(compiler.compile_pending() for _ in range(3)))
costs: Final = _rows(
ledger_database, "SELECT payload::text FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'"
)
assert len(costs) == 1
cost_envelope: Final = json.loads(str(costs[0][0]))
committed: Final = CostEvent.model_validate_json(json.dumps(cost_envelope["event"]))
assert committed.snapshot.total_cost == Decimal(headers["x-litellm-response-cost"])
assert float(committed.snapshot.total_cost) == pytest.approx(0.21)
monkeypatch.setattr(litellm, "model_cost", {"openai/capture-priced-model": {"input_cost_per_token": 99.0}})
monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.99})
monkeypatch.setattr(litellm, "cost_margin_config", {"global": {"fixed_amount": 999.0}})
replayed: Final = price_usage(json.dumps(cost_envelope["pricing_inputs"]))
assert replayed == committed.snapshot
assert litellm.model_cost["openai/capture-priced-model"]["input_cost_per_token"] == 99.0
await capture_proxy_response(logger, response)
await compiler.compile_pending()
assert _rows(ledger_database, "SELECT count(*) FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [(1,)]
finally:
release.set()
await runtime.stop(5)
@pytest.mark.asyncio
@pytest.mark.parametrize("conflicting_result", (False, True))
async def test_failing_compilation_page_does_not_starve_later_usage(
writer: _Writer, ledger_database: str, conflicting_result: bool
) -> None:
_, store = writer
broken: Final = _usage("broken-price")
healthy: Final = _usage("healthy-price")
assert isinstance(broken, OutboundUsageEvent)
assert isinstance(healthy, OutboundUsageEvent)
await _insert(store, broken)
await _insert(store, healthy)
def price(envelope: str) -> PricingSnapshot | Unpriced:
if OutboundUsageEvent.model_validate(json.loads(envelope)["event"]).event_id == broken.event_id:
if conflicting_result:
return Unpriced(usage_event_id="different-usage", reason="usage_unknown")
raise RuntimeError("temporary calculator failure")
return _cost().snapshot
await LoggingCostCompiler(store, price, _TARGETS).compile_pending(limit=1)
assert await store.pending_compilation(limit=1) == (healthy,)
_rows(
ledger_database,
'UPDATE "LiteLLM_AccountingEvent" SET compilation_retry_at = CURRENT_TIMESTAMP WHERE event_id = %s',
(broken.event_id,),
)
await LoggingCostCompiler(store, price, _TARGETS).compile_pending(limit=1)
assert _rows(ledger_database, "SELECT source_event_id FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'") == [
(healthy.event_id,)
]
assert isinstance(await LoggingCostCompiler(store, lambda _: _cost().snapshot, _TARGETS).compile(broken), CostEvent)
assert await store.pending_compilation() == ()
@pytest.mark.asyncio
@pytest.mark.parametrize("summary_first", (False, True))
async def test_batch_callbacks_commit_only_row_charges_across_restart(
writer: _Writer, ledger_database: str, monkeypatch: pytest.MonkeyPatch, summary_first: bool
) -> None:
import litellm
from litellm.proxy.spend_tracking.usage_capture import (
UsageBatchWriter,
capture_batch_callback,
capture_callback,
capture_logging_usage,
)
from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings
from tests.unit.proxy.spend_tracking.test_usage_capture import _logger
from tests.unit.proxy.spend_tracking.test_usage_meter import _manual_flush
_, store = writer
rates: Final = {
"input_cost_per_token_batches": 0.003,
"output_cost_per_token_batches": 0.004,
"litellm_provider": "openai",
"mode": "chat",
}
monkeypatch.setitem(litellm.model_cost, "capture-model", rates)
usage: Final = {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
response: Final = {
"id": "managed-job",
"status": "completed",
"usage": {
"batch_usage_by_model": {"capture-model": usage},
"batch_usage_by_request": [{"model": "capture-model", "usage": usage} for _ in range(2)],
},
}
logger: Final = _logger("aretrieve_batch")
batches: Final = UsageBatchWriter(store.insert_usage_batch, store.load_usage_batch)
meter: Final = DurableUsageMeter(batches.write, UsageCaptureSettings(), wait_for_batch=_manual_flush)
compiler: Final = LoggingCostCompiler(store)
capture_logging_usage(meter, logger, response, _AT, _AT, freeze_only=True)
kwargs: Final = {**logger.model_call_details, "litellm_logging_obj": logger}
async def summary() -> None:
capture_callback(
meter,
kwargs,
{
"id": "provider-job",
"status": "completed",
"usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30},
},
_AT,
_AT,
)
await meter.flush()
await compiler.compile_pending()
try:
if summary_first:
await summary()
await capture_batch_callback(meter, store.load_usage_batch, kwargs, response, _AT, _AT)
await compiler.compile_pending()
if not summary_first:
await summary()
before: Final = _rows(
ledger_database, 'SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent" ORDER BY event_id'
)
costs: Final = _rows(
ledger_database, "SELECT payload::text FROM \"LiteLLM_AccountingEvent\" WHERE kind = 'cost'"
)
deliveries: Final = _rows(
ledger_database,
'SELECT event_id, target, state FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target',
)
assert len(costs) == 2
snapshots: Final = tuple(
CostEvent.model_validate_json(json.dumps(json.loads(str(row[0]))["event"])).snapshot for row in costs
)
assert sum(float(snapshot.total_cost) for snapshot in snapshots) == pytest.approx(2 * (10 * 0.003 + 5 * 0.004))
monkeypatch.setitem(litellm.model_cost, "capture-model", {**rates, "input_cost_per_token_batches": 99.0})
async with connected_proxy_prisma(ledger_database) as restarted:
ledger: Final = AccountingLedger.for_client(restarted)
from litellm.proxy.openai_files_endpoints.common_utils import encode_file_id_with_model
for job_id in ("managed-job", encode_file_id_with_model("managed-job", "capture-model", "batch")):
await capture_batch_callback(
meter, ledger.load_usage_batch, kwargs, {**response, "id": job_id}, _AT, _AT
)
await asyncio.gather(*(LoggingCostCompiler(ledger).compile_pending() for _ in range(2)))
assert (
_rows(ledger_database, 'SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent" ORDER BY event_id')
== before
)
assert (
_rows(
ledger_database,
'SELECT event_id, target, state FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target',
)
== deliveries
)
for row, snapshot in zip(costs, snapshots, strict=True):
assert price_usage(json.dumps(json.loads(str(row[0]))["pricing_inputs"])) == snapshot
finally:
await meter.stop(1)

View file

@ -549,7 +549,7 @@ async def test_completion_sagemaker(
assert route.called
assert response.choices[0].message.content == "SageMaker reply"
assert litellm.completion_cost(completion_response=response) == pytest.approx(rate)
assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate)
@pytest.mark.asyncio

File diff suppressed because it is too large Load diff

View file

@ -1,13 +1,13 @@
import asyncio
import json
from collections.abc import Awaitable, Mapping
from collections.abc import Mapping
from datetime import datetime, timezone
from decimal import Decimal
from typing import Final
import pytest
from litellm.accounting import Metric, MetricName, Outcome
from litellm.accounting import Metric, MetricName, OutboundUsageEvent, Outcome, Unpriced
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite
from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureError, UsageCaptureSettings
@ -334,7 +334,7 @@ async def test_competing_writers_reuse_only_terminal_pricing_snapshots(terminal:
@pytest.mark.asyncio
async def test_batch_poll_has_distinct_identity_and_only_terminal_usage_is_chargeable() -> None:
async def test_batch_poll_and_terminal_summary_are_not_charge_owners() -> None:
from litellm.proxy.spend_tracking.usage_capture import capture_callback
database: Final = StoredUsage()
@ -352,8 +352,8 @@ async def test_batch_poll_has_distinct_identity_and_only_terminal_usage_is_charg
await meter.close()
assert pending.writes[-1].event.event_id == "poll:status"
assert json.loads(pending.writes[-1].envelope)["charge_eligible"] is False
assert completed.writes[-1].event.event_id == "batch_batch_cost"
assert json.loads(completed.writes[-1].envelope)["charge_eligible"] is True
assert completed.writes[-1].event.event_id == "batch_batch_cost:summary"
assert json.loads(completed.writes[-1].envelope)["charge_eligible"] is False
assert len(completed.writes) == 1
@ -441,13 +441,13 @@ async def test_supported_sync_ack_waits_for_commit(call_type: str, monkeypatch:
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True})
deadlines: Final = asyncio.Queue[float | None]()
original_wait: Final = asyncio.wait_for
original_timeout: Final = asyncio.timeout
async def wait_with_deadline(awaitable: Awaitable[object], timeout: float | None) -> object:
deadlines.put_nowait(timeout)
return await original_wait(awaitable, timeout)
def timeout_with_deadline(delay: float | None) -> asyncio.Timeout:
deadlines.put_nowait(delay)
return original_timeout(delay)
monkeypatch.setattr(asyncio, "wait_for", wait_with_deadline)
monkeypatch.setattr(asyncio, "timeout", timeout_with_deadline)
response: Final = asyncio.create_task(
capture_proxy_response(
_logger(call_type),
@ -1021,9 +1021,8 @@ async def test_batch_capture_provenance_tracks_observations_not_envelope_presenc
envelope: Final = json.loads(database.rows[-1].envelope)
assert envelope["usage_source"] == ("unknown" if measured is None else "measured")
assert envelope["usage"]["batch_usage_by_model"] == usage["batch_usage_by_model"]
assert database.rows[-1].event.metrics == (
None if measured is None else (Metric(name=MetricName.TOTAL_TOKENS, value=Decimal(measured["total_tokens"])),)
)
assert database.rows[-1].event.metrics is None
assert envelope["charge_eligible"] is False
@pytest.mark.asyncio
@ -1075,17 +1074,14 @@ async def test_response_ack_timeout_preserves_success_and_later_commits(monkeypa
runtime.start(database.write, UsageCaptureSettings(batch_size=2))
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
monkeypatch.setattr(proxy_server, "prisma_client", None)
monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True})
monkeypatch.setattr(
proxy_server, "general_settings", {"accounting_sync_ack": True, "accounting_sync_ack_timeout_seconds": 0.01}
)
logger: Final = _logger()
async def expired_deadline(future: object, timeout: float) -> None:
raise TimeoutError("controlled response deadline")
try:
with monkeypatch.context() as clock:
clock.setattr(asyncio, "wait_for", expired_deadline)
with pytest.raises(HTTPException) as error:
await capture_proxy_response(logger, ModelResponse(id="provider", usage=Usage(prompt_tokens=3)))
with pytest.raises(HTTPException) as error:
await capture_proxy_response(logger, ModelResponse(id="provider", usage=Usage(prompt_tokens=3)))
assert error.value.status_code == 503
await database.entered.wait()
captured: Final = logger.model_call_details["_accounting_capture"]
@ -1710,3 +1706,80 @@ async def test_batch_replay_reserves_the_accepted_pricing_snapshot_size() -> Non
assert runtime.meter is not None and runtime.meter.pending_bytes == 0
finally:
await runtime.stop(1)
class DelayedCostCompiler:
def __init__(self) -> None:
self.entered: Final = asyncio.Event()
self.cancelled: Final = asyncio.Event()
self.release: Final = asyncio.Event()
async def compile(self, event: OutboundUsageEvent) -> Unpriced:
self.entered.set()
try:
await self.release.wait()
except asyncio.CancelledError:
self.cancelled.set()
raise
return Unpriced(usage_event_id=event.event_id, reason="usage_unknown")
@pytest.mark.asyncio
async def test_sync_ack_preserves_committed_usage_ack_when_compilation_times_out(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.proxy import proxy_server
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response
from litellm.types.utils import ModelResponse, Usage
database: Final = CommitBarrier()
database.release.set()
compiler: Final = DelayedCostCompiler()
runtime: Final = UsageCaptureRuntime()
runtime.start(database.write, UsageCaptureSettings(batch_size=2), compiler=compiler)
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
monkeypatch.setattr(
proxy_server, "general_settings", {"accounting_sync_ack": True, "accounting_sync_ack_timeout_seconds": 0.1}
)
try:
await asyncio.wait_for(
capture_proxy_response(_logger(), ModelResponse(id="provider", usage=Usage(prompt_tokens=3))), 2
)
assert compiler.entered.is_set()
assert compiler.cancelled.is_set()
assert database.committed == ("caller:inbound", "provider")
finally:
compiler.release.set()
await runtime.stop(1)
@pytest.mark.asyncio
@pytest.mark.parametrize("call_type", ["create_batch", "acreate_batch"])
async def test_batch_creation_is_not_a_charge_owner(call_type: str) -> None:
from litellm.proxy.spend_tracking.cost_compiler import price_usage
from litellm.proxy.spend_tracking.usage_capture import capture_callback
from litellm.accounting import Unpriced
database: Final = StoredUsage()
meter: Final = DurableUsageMeter(database.insert, UsageCaptureSettings(), wait_for_batch=_manual_flush)
try:
captured: Final = capture_callback(
meter,
{
"call_type": call_type,
"litellm_call_id": "create",
"model": "capture-model",
"litellm_params": {},
},
{"id": "batch", "status": "validating"},
NOW,
NOW,
)
await meter.close()
outbound: Final = captured.writes[-1]
assert price_usage(outbound.envelope) == Unpriced(
usage_event_id=outbound.event.event_id, reason="not_charge_owner"
)
finally:
await meter.stop(1)

View file

@ -11108,3 +11108,50 @@ def test_upstream_cost_header_total_on_an_unmapped_alias_is_split_by_the_served_
input_share: Final = (10 * 0.001) / (10 * 0.001 + 20 * 0.002)
assert breakdown["input_cost"] == pytest.approx(0.25 * input_share)
assert breakdown["output_cost"] == pytest.approx(0.25 * (1 - input_share))
@pytest.mark.parametrize(
("model", "retained_key", "deployment_rate"),
(
("video-model", "openai/video-model", None),
("openai/video-model", "video-model", None),
("openai/video-model", "openai/video-model", 0.7),
),
)
def test_video_pricing_uses_retained_catalog_and_deployment_precedence(
monkeypatch: pytest.MonkeyPatch, model: str, retained_key: str, deployment_rate: float | None
) -> None:
from pydantic import TypeAdapter
from litellm.cost_calculator import default_video_cost_calculator
from litellm.litellm_core_utils.llm_cost_calc.pricing_context import RetainedPricing, retained_pricing
from litellm.types.utils import ModelInfo
info: Final = TypeAdapter(ModelInfo).validate_python(
{
"key": retained_key,
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
"input_cost_per_token": None,
"output_cost_per_token": None,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_openai_params": None,
"output_cost_per_video_per_second": 0.2,
}
)
retained: Final = RetainedPricing(json.dumps({retained_key: info}), "{}", "{}")
changed: Final = {**info, "output_cost_per_video_per_second": 9.0}
monkeypatch.setattr(litellm, "model_cost", {"video-model": changed, "openai/video-model": changed})
deployment: Final = (
TypeAdapter(ModelInfo).validate_python({**info, "output_cost_per_video_per_second": deployment_rate})
if deployment_rate is not None
else None
)
with retained_pricing(retained):
price: Final = default_video_cost_calculator(
model, duration_seconds=4, custom_llm_provider="openai", model_info=deployment
)
assert price == pytest.approx(4 * (deployment_rate if deployment_rate is not None else 0.2))
assert litellm.model_cost["video-model"]["output_cost_per_video_per_second"] == 9.0

View file

@ -1,13 +1,4 @@
"""
Test that register_model() in completion() and embedding() passes all
custom pricing fields from kwargs and model_info, not just the base
input/output costs.
Previously, only input_cost_per_token, output_cost_per_token, and
litellm_provider were forwarded. Fields like cache_read_input_token_cost,
mode, and supports_prompt_caching were dropped, causing incorrect cost
calculations for DB-sourced models with prompt caching pricing.
"""
"""Verify explicit model registration and isolated request pricing preserve effective rates."""
import copy
import itertools
@ -17,9 +8,9 @@ from typing import Final, NoReturn
import pytest
import litellm
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.main import _build_custom_pricing_entry
from litellm.utils import _invalidate_model_cost_lowercase_map
@ -646,17 +637,15 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
"""
shared_key = "openai/text-embedding-3-small"
deployment_id = "lit3991-wildcard-embed-zero"
snapshot = _snapshot_model_cost_entries(
[shared_key, "text-embedding-3-small", deployment_id]
)
builtin_input_cost = litellm.get_model_info(model=shared_key)[
"input_cost_per_token"
]
snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id])
builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"]
assert builtin_input_cost > 0
logger: Final = _request_logger(shared_key, "embedding")
try:
litellm.embedding(
response = litellm.embedding(
model=shared_key,
litellm_logging_obj=logger,
input=["hello"],
api_key="fake-key",
input_cost_per_token=0.0,
@ -666,12 +655,10 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
mock_response=[0.1, 0.2],
)
assert (
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
== builtin_input_cost
), "wildcard deployment's zero pricing leaked into the shared model_cost key"
assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0
assert litellm.model_cost[deployment_id]["output_cost_per_token"] == 0.0
assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, (
"wildcard deployment's zero pricing leaked into the shared model_cost key"
)
assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == 0.0
sibling_response = litellm.embedding(
model=shared_key,
@ -679,9 +666,7 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
api_key="fake-key",
mock_response=[0.1, 0.2],
)
sibling_cost = litellm.completion_cost(
completion_response=sibling_response, call_type="embedding"
)
sibling_cost = litellm.completion_cost(completion_response=sibling_response, call_type="embedding")
assert sibling_cost == pytest.approx(10 * builtin_input_cost)
finally:
_restore_model_cost_entries(snapshot)
@ -695,17 +680,15 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
shared_key = "openai/text-embedding-3-small"
deployment_id = "lit3991-wildcard-embed-custom"
override_input_cost = 5e-05
snapshot = _snapshot_model_cost_entries(
[shared_key, "text-embedding-3-small", deployment_id]
)
builtin_input_cost = litellm.get_model_info(model=shared_key)[
"input_cost_per_token"
]
snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id])
builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"]
assert builtin_input_cost != override_input_cost
logger: Final = _request_logger(shared_key, "embedding")
try:
response = litellm.embedding(
model=shared_key,
litellm_logging_obj=logger,
input=["hello"],
api_key="fake-key",
input_cost_per_token=override_input_cost,
@ -715,19 +698,9 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
mock_response=[0.1, 0.2],
)
request_cost = litellm.completion_cost(
completion_response=response,
model=shared_key,
custom_llm_provider="openai",
call_type="embedding",
custom_pricing=True,
router_model_id=deployment_id,
)
request_cost = logger.response_cost_calculator(response, ignore_stamped_cost=True)
assert request_cost == pytest.approx(10 * override_input_cost)
assert (
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
== builtin_input_cost
)
assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost
finally:
_restore_model_cost_entries(snapshot)
@ -736,17 +709,15 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing():
"""Same isolation as the embedding path, exercised through completion()."""
shared_key = "openai/gpt-4o-mini"
deployment_id = "lit3991-wildcard-chat-zero"
snapshot = _snapshot_model_cost_entries(
[shared_key, "gpt-4o-mini", deployment_id]
)
builtin_input_cost = litellm.get_model_info(model=shared_key)[
"input_cost_per_token"
]
snapshot = _snapshot_model_cost_entries([shared_key, "gpt-4o-mini", deployment_id])
builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"]
assert builtin_input_cost > 0
logger: Final = _request_logger(shared_key, "completion")
try:
litellm.completion(
response = litellm.completion(
model=shared_key,
litellm_logging_obj=logger,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
input_cost_per_token=0.0,
@ -756,47 +727,43 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing():
mock_response="hello back",
)
assert (
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
== builtin_input_cost
), "wildcard deployment's zero pricing leaked into the shared model_cost key"
assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0
assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, (
"wildcard deployment's zero pricing leaked into the shared model_cost key"
)
assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == 0.0
finally:
_restore_model_cost_entries(snapshot)
def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key():
"""Direct SDK calls (no router deployment id in metadata) keep the legacy
behavior: custom pricing is registered under ``{provider}/{model}`` and the
request is costed with it.
"""
model_key = "openai/lit3991-direct-sdk-embed-model"
override_input_cost = 3e-05
try:
response = litellm.embedding(
model=model_key,
input=["hello"],
api_key="fake-key",
input_cost_per_token=override_input_cost,
output_cost_per_token=override_input_cost * 2,
mock_response=[0.1, 0.2],
)
def _request_logger(model: str, call_type: str) -> Logging:
from datetime import datetime, timezone
assert (
litellm.model_cost[model_key]["input_cost_per_token"]
== override_input_cost
)
cost = litellm.completion_cost(
completion_response=response,
model=model_key,
custom_llm_provider="openai",
call_type="embedding",
custom_pricing=True,
)
assert cost == pytest.approx(10 * override_input_cost)
finally:
litellm.model_cost.pop(model_key, None)
_invalidate_model_cost_lowercase_map()
return Logging(
model=model,
messages=[],
stream=False,
call_type=call_type,
start_time=datetime.now(timezone.utc),
litellm_call_id="override",
function_id="override",
)
def test_embedding_direct_sdk_custom_pricing_is_request_scoped() -> None:
model: Final = "openai/request-priced-embedding"
before: Final = _snapshot_model_cost_entries((model,))
logger: Final = _request_logger(model, "embedding")
response: Final = litellm.embedding(
model=model,
input=["hello"],
api_key="fake-key",
input_cost_per_token=0.003,
output_cost_per_token=0.006,
mock_response=[0.1, 0.2],
litellm_logging_obj=logger,
)
assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(10 * 0.003)
assert _snapshot_model_cost_entries((model,)) == before
@pytest.mark.parametrize("route", ["", "converse/", "invoke/"])
@ -1407,10 +1374,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing():
object stays untouched, and the caller's incoming nested dict is never
inserted by reference into the merged result.
"""
from litellm.utils import update_dictionary
from typing import cast
from litellm.utils import update_dictionary
existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"})
existing = {"off_peak_pricing": existing_nested}
incoming_nested = cast(
@ -1609,22 +1576,24 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing():
del router
def test_completion_registers_cost_per_second_pricing():
model_key: Final = "openai/test-cost-per-second-registration"
original_entries: Final = _snapshot_model_cost_entries([model_key])
def test_completion_per_second_pricing_is_request_scoped() -> None:
from litellm.types.utils import ModelResponse, Usage
try:
litellm.completion(
model=model_key,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
cost_per_second=0.02,
mock_response="hello back",
)
assert litellm.model_cost[model_key]["cost_per_second"] == 0.02
finally:
_restore_model_cost_entries(original_entries)
model: Final = "openai/request-priced-seconds"
logger: Final = _request_logger(model, "completion")
before: Final = _snapshot_model_cost_entries((model,))
litellm.completion(
model=model,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
cost_per_second=0.02,
mock_response="hello back",
litellm_logging_obj=logger,
)
response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=0, completion_tokens=0))
response._response_ms = 2000
assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(0.04)
assert _snapshot_model_cost_entries((model,)) == before
def test_update_model_cost():
@ -1676,31 +1645,346 @@ def test_register_model_scopes_builtin_match_to_the_given_provider():
_restore_model_cost_entries(model_cost_entries)
def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider():
from litellm.main import _register_custom_pricing_for_request
def test_per_request_custom_pricing_preserves_a_colliding_deployment() -> None:
from litellm.main import _set_request_pricing
from litellm.types.utils import ModelResponse, Usage
deployment_id: Final = "baseten/zai-org/glm-5.2"
builtin_key: Final = "baseten/zai-org/GLM-5.2"
shared_key: Final = "openai/zai-org/GLM-5.2"
model_cost_entries: Final = _snapshot_model_cost_entries((deployment_id, builtin_key, shared_key))
builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key])
openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models)
try:
_register_custom_pricing_for_request(
model="zai-org/GLM-5.2",
custom_llm_provider="openai",
kwargs={
"input_cost_per_token": 0.00000096,
"output_cost_per_token": 0.00000302,
"metadata": {"model_info": {"id": deployment_id}},
},
model_info={"mode": "chat"},
deployment: Final = "baseten/zai-org/glm-5.2"
model: Final = "zai-org/GLM-5.2"
keys: Final = (deployment, "baseten/zai-org/GLM-5.2", f"openai/{model}")
before: Final = _snapshot_model_cost_entries(keys)
logger: Final = _request_logger(model, "completion")
logger.update_environment_variables(
model=model, user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai"
)
_set_request_pricing(
logger,
model,
"openai",
{"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "metadata": {"model_info": {"id": deployment}}},
{"mode": "chat"},
)
response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=5))
assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(0.02)
assert _snapshot_model_cost_entries(keys) == before
@pytest.mark.parametrize("call_type", ("completion", "embedding"))
def test_concurrent_requests_keep_distinct_prices_and_leave_default_rates(call_type: str) -> None:
from concurrent.futures import ThreadPoolExecutor
from threading import Barrier
model: Final = "openai/gpt-4o-mini" if call_type == "completion" else "openai/text-embedding-3-small"
before: Final = _snapshot_model_cost_entries((model, model.removeprefix("openai/")))
ready: Final = Barrier(2)
def execute(logger: Logging, rate: float) -> object:
if call_type == "completion":
return litellm.completion(
model=model,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
input_cost_per_token=rate,
output_cost_per_token=rate,
mock_response="hello",
litellm_logging_obj=logger,
)
return litellm.embedding(
model=model,
input=["hello"],
api_key="fake-key",
input_cost_per_token=rate,
output_cost_per_token=rate,
mock_response=[0.1],
litellm_logging_obj=logger,
)
entry: Final = litellm.model_cost[deployment_id]
assert entry["input_cost_per_token"] == 0.00000096
assert entry["output_cost_per_token"] == 0.00000302
assert litellm.model_cost[builtin_key] == builtin_row_before
def price(rate: float) -> float | None:
logger: Final = _request_logger(model, call_type)
response: Final = execute(logger, rate)
ready.wait(timeout=10)
return logger.response_cost_calculator(response, ignore_stamped_cost=True)
with ThreadPoolExecutor(max_workers=2) as pool:
amounts: Final = tuple(pool.map(price, (0.001, 0.003)))
assert amounts[0] is not None and amounts[0] > 0
assert amounts[1] == pytest.approx(amounts[0] * 3)
assert _snapshot_model_cost_entries((model, model.removeprefix("openai/"))) == before
def test_request_pricing_preserves_reasoning_and_cache_rates() -> None:
from litellm.types.utils import ModelResponse, Usage
model: Final = "openai/gpt-4o-mini"
logger: Final = _request_logger(model, "completion")
litellm.completion(
model=model,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
input_cost_per_token=0.01,
output_cost_per_token=0.02,
cache_read_input_token_cost=0.001,
output_cost_per_reasoning_token=0.04,
mock_response="hello",
litellm_logging_obj=logger,
)
response: Final = ModelResponse(
model="gpt-4o-mini",
usage=Usage(
prompt_tokens=100,
completion_tokens=20,
prompt_tokens_details={"cached_tokens": 40},
completion_tokens_details={"reasoning_tokens": 5},
),
)
assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(
60 * 0.01 + 40 * 0.001 + 15 * 0.02 + 5 * 0.04
)
@pytest.mark.parametrize("explicit_null_mode", (False, True))
def test_router_custom_token_prices_accept_missing_deployment_mode(explicit_null_mode: bool) -> None:
from litellm import Router
from litellm.types.utils import ModelResponse
deployment_id: Final = "custom-pricing-without-mode"
keys: Final = ("gpt-4o-mini", "openai/gpt-4o-mini", deployment_id)
original: Final = _snapshot_model_cost_entries(keys)
model_info: Final = {"id": deployment_id, **({"mode": None} if explicit_null_mode else {})}
try:
router: Final = Router(
model_list=[
{
"model_name": "priced-chat",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "fake-key",
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
"mock_response": "hello back",
},
"model_info": model_info,
}
],
num_retries=0,
)
registered: Final = _snapshot_model_cost_entries(keys)
logger: Final = _request_logger("openai/gpt-4o-mini", "completion")
response: Final = router.completion(
model="priced-chat", messages=[{"role": "user", "content": "hello"}], litellm_logging_obj=logger
)
assert isinstance(response, ModelResponse)
assert response.usage.prompt_tokens > 0
assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(
response.usage.prompt_tokens * 0.001 + response.usage.completion_tokens * 0.002
)
assert _snapshot_model_cost_entries(keys) == registered
finally:
_restore_model_cost_entries(model_cost_entries)
litellm.open_ai_chat_completion_models.intersection_update(openai_models_before)
_restore_model_cost_entries(original)
def test_changed_request_flat_rate_preserves_explicit_deployment_tier_prices() -> None:
from litellm.types.utils import ModelResponse, Usage
deployment_id: Final = "request-price-variants"
original: Final = _snapshot_model_cost_entries((deployment_id,))
variants: Final = {"input_cost_per_token_priority": 0.03, "output_cost_per_token_priority": 0.04}
try:
litellm.register_model(
{
deployment_id: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 0.01,
"output_cost_per_token": 0.02,
**variants,
}
},
persist_across_reloads=False,
)
logger: Final = _request_logger("openai/gpt-4o-mini", "completion")
logger.update_environment_variables(
model="gpt-4o-mini", custom_llm_provider="openai", litellm_params={}, optional_params={}
)
logger.set_request_pricing(
"gpt-4o-mini",
"openai",
{"input_cost_per_token": 0.05, "output_cost_per_token": 0.02, **variants},
deployment_id,
)
response: Final = ModelResponse(
model="gpt-4o-mini",
service_tier="priority",
usage=Usage(prompt_tokens=10, completion_tokens=4, total_tokens=14),
)
assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(
10 * 0.03 + 4 * 0.04
)
finally:
_restore_model_cost_entries(original)
@pytest.mark.parametrize("call_type", ("completion", "embedding"))
def test_sdk_followup_prices_change_only_after_explicit_registration(call_type: str) -> None:
model: Final = f"openai/followup-{call_type}-prices"
original: Final = _snapshot_model_cost_entries((model, model.removeprefix("openai/")))
models_before: Final = frozenset(litellm.open_ai_chat_completion_models)
logger: Final = _request_logger(model, call_type)
try:
litellm.register_model(
{
model: {
"litellm_provider": "openai",
"mode": "chat" if call_type == "completion" else "embedding",
"input_cost_per_token": 0.0001,
"output_cost_per_token": 0.0002,
}
},
persist_across_reloads=False,
)
def request(prices: dict[str, float], *, logging_obj: Logging | None = None) -> object:
if call_type == "completion":
return litellm.completion(
model=model,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
mock_response="hello back",
litellm_logging_obj=logging_obj,
**prices,
)
return litellm.embedding(
model=model,
input=["hello"],
api_key="fake-key",
mock_response=[0.1, 0.2],
litellm_logging_obj=logging_obj,
**prices,
)
first: Final = request({"input_cost_per_token": 0.001, "output_cost_per_token": 0.002}, logging_obj=logger)
second: Final = request({})
assert logger.response_cost_calculator(first, ignore_stamped_cost=True) == pytest.approx(
0.05 if call_type == "completion" else 0.01
)
assert litellm.completion_cost(completion_response=second, model=model, call_type=call_type) == pytest.approx(
0.005 if call_type == "completion" else 0.001
)
assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001
litellm.register_model(
{model: {"input_cost_per_token": 0.0003, "output_cost_per_token": 0.0006}},
persist_across_reloads=False,
)
third: Final = request({})
assert litellm.completion_cost(completion_response=third, model=model, call_type=call_type) == pytest.approx(
0.015 if call_type == "completion" else 0.003
)
assert logger.response_cost_calculator(first, ignore_stamped_cost=True) == pytest.approx(
0.05 if call_type == "completion" else 0.01
)
finally:
_restore_model_cost_entries(original)
litellm.open_ai_chat_completion_models.intersection_update(models_before)
@pytest.mark.asyncio
@pytest.mark.parametrize("call_type", ("completion", "acompletion", "embedding", "aembedding"))
@pytest.mark.parametrize("custom_primary", (True, False))
async def test_router_fallback_prices_belong_to_the_served_attempt(
monkeypatch: pytest.MonkeyPatch, call_type: str, custom_primary: bool
) -> None:
import httpx
import respx
from datetime import datetime, timezone
model: Final = "openai/fallback-price-regression"
embedding: Final = "embedding" in call_type
keys: Final = (model, model.removeprefix("openai/"), "price-primary", "price-fallback")
original: Final = _snapshot_model_cost_entries(keys)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "cost_discount_config", {})
monkeypatch.setattr(litellm, "cost_margin_config", {})
try:
litellm.register_model(
{
model: {
"litellm_provider": "openai",
"mode": "embedding" if embedding else "chat",
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
}
},
persist_across_reloads=False,
)
router: Final = litellm.Router(
model_list=[
{
"model_name": name,
"litellm_params": {
"model": model,
"api_key": "fake-key",
"api_base": f"https://{name}.example/v1",
**({"input_cost_per_token": rate, "output_cost_per_token": rate} if rate is not None else {}),
},
"model_info": {"id": f"price-{name}"},
}
for name, rate in (
("primary", 0.0 if custom_primary else None),
("fallback", None if custom_primary else 0.003),
)
],
fallbacks=[{"primary": ["fallback"]}],
num_retries=0,
)
registered: Final = _snapshot_model_cost_entries(keys)
logger: Final = Logging(
model=model,
messages=[],
stream=False,
call_type=call_type,
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
litellm_call_id="fallback-pricing",
function_id="fallback-pricing",
)
endpoint: Final = "embeddings" if embedding else "chat/completions"
body: Final = (
{
"object": "list",
"model": model.removeprefix("openai/"),
"data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}],
"usage": {"prompt_tokens": 10, "total_tokens": 10},
}
if embedding
else {
"id": "fallback-result",
"object": "chat.completion",
"created": 1,
"model": model.removeprefix("openai/"),
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
)
with respx.mock(assert_all_called=True) as wire:
primary: Final = wire.post(f"https://primary.example/v1/{endpoint}").mock(
return_value=httpx.Response(429, json={"error": {"message": "rate limit", "type": "rate_limit_error"}})
)
fallback: Final = wire.post(f"https://fallback.example/v1/{endpoint}").mock(
return_value=httpx.Response(200, json=body)
)
request: Final = {"model": "primary", "litellm_logging_obj": logger}
if call_type == "acompletion":
response = await router.acompletion(**request, messages=[{"role": "user", "content": "probe"}])
elif call_type == "completion":
response = router.completion(**request, messages=[{"role": "user", "content": "probe"}])
elif call_type == "aembedding":
response = await router.aembedding(**request, input=["probe"])
else:
response = router.embedding(**request, input=["probe"])
expected: Final = (0.01 if embedding else 0.02) if custom_primary else (0.03 if embedding else 0.045)
assert logger.response_cost_calculator(response, ignore_stamped_cost=True) == pytest.approx(expected)
assert response._hidden_params["response_cost"] == pytest.approx(expected)
assert primary.called
assert fallback.call_count == 1
assert _snapshot_model_cost_entries(keys) == registered
finally:
_restore_model_cost_entries(original)

View file

@ -2333,7 +2333,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None:
@respx.mock
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
def test_completion_registers_test_owned_model_prices() -> None:
def test_completion_keeps_custom_prices_scoped_to_the_request() -> None:
model: Final = "migration-owned-pricing-model"
messages: Final = [{"role": "user", "content": "price control"}]
litellm.register_model(
@ -2376,8 +2376,8 @@ def test_completion_registers_test_owned_model_prices() -> None:
)
assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages}
assert litellm.model_cost[model]["input_cost_per_token"] == 0.001
assert litellm.model_cost[model]["output_cost_per_token"] == 0.002
assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001
assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002
assert response._hidden_params["response_cost"] == pytest.approx(0.018)