scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

feat(proxy): price cache and reasoning tokens in /cost/estimate

Browse source 
POST /cost/estimate now accepts cache_read_input_tokens,
cache_creation_input_tokens and reasoning_tokens, bills them at the
model's cache and reasoning rates, and reports each share per request,
per day and per month next to the rates it used.

Custom-priced deployments also get cache and reasoning lines in the cost
breakdown now, so the estimate and the spend logs reconcile with their
totals instead of showing zero for those tokens.

Requested by a customer (Pylon #7365).

Claude-Session: https://claude.ai/code/session_011Tn3657NkV6ojLqewL64Kb
This commit is contained in:
ryan-crabbe-berri 2026-09-07 16:00:20 -07:00
parent 38683643e0
commit c84131b81a
8 changed files with 641 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/cost_calculator.py
View file

@ -1746,6 +1746,7 @@ def completion_cost(
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
custom_cost_per_token=custom_cost_per_token,
)
_reasoning_cost = _token_type_breakdown.reasoning_cost
_cache_read_cost = _token_type_breakdown.cache_read_cost

68
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -19,6 +19,7 @@ from litellm.types.utils import (
CacheCreationTokenDetails,
CallTypes,
CompletionTokensDetailsWrapper,
CostPerToken,
DataResidency,
ImageResponse,
ModelInfo,
@ -1307,6 +1308,40 @@ class TokenTypeCostBreakdown:
cache_creation_cost: float
def _reasoning_token_count(usage: Usage) -> int:
parsed: Final = (
parse_completion_tokens_details(usage)["reasoning_tokens"] if usage.completion_tokens_details is not None else 0
)
return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetails | None]:
"""(cache read tokens, cache creation tokens, cache creation details): read from prompt_tokens_details
first, then the private top-level counters the Usage constructor mirrors cache tokens onto for
providers/callers that bypass the details."""
parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None
parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0
parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0
return (
parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)),
parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)),
parsed["cache_creation_token_details"] if parsed is not None else None,
)
def _custom_pricing_token_type_breakdown(usage: Usage, custom_cost_per_token: CostPerToken) -> TokenTypeCostBreakdown:
"""Flat custom pricing has no tiers, uplifts or reasoning rate: cache tokens bill at the configured
cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does."""
input_rate: Final = custom_cost_per_token["input_cost_per_token"]
cache_read_tokens, cache_creation_tokens, _ = _cache_token_counts(usage)
return TokenTypeCostBreakdown(
reasoning_cost=float(_reasoning_token_count(usage)) * custom_cost_per_token["output_cost_per_token"],
cache_read_cost=float(cache_read_tokens) * custom_cost_per_token.get("cache_read_input_token_cost", input_rate),
cache_creation_cost=float(cache_creation_tokens)
* custom_cost_per_token.get("cache_creation_input_token_cost", input_rate),
)
def get_token_type_cost_breakdown(
model: str,
custom_llm_provider: str | None,
@ -1315,6 +1350,7 @@ def get_token_type_cost_breakdown(
data_residency: str | None = None,
vertex_location: str | None = None,
current_time: datetime | None = None,
custom_cost_per_token: CostPerToken | None = None,
) -> TokenTypeCostBreakdown:
"""
Provider-agnostic cost of reasoning and cache tokens, derived from the usage
@ -1325,9 +1361,13 @@ def get_token_type_cost_breakdown(
land on ``prompt_tokens_details`` (via the Usage constructor and provider
transformations) and reasoning tokens on ``completion_tokens_details``. It reuses
the same rate-resolution primitives as the total-cost path so the breakdown can
never drift from the totals. Returns zeros (never raises) when the model or its
pricing cannot be resolved.
never drift from the totals. A deployment billed by ``custom_cost_per_token`` is
priced from those flat rates instead of the cost map, for the same reason.
Returns zeros (never raises) when the model or its pricing cannot be resolved.
"""
if custom_cost_per_token is not None:
return _custom_pricing_token_type_breakdown(usage=usage, custom_cost_per_token=custom_cost_per_token)
try:
model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception:
@ -1348,12 +1388,6 @@ def get_token_type_cost_breakdown(
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
)
reasoning_tokens = (
parse_completion_tokens_details(usage)["reasoning_tokens"] if usage.completion_tokens_details is not None else 0
)
if not reasoning_tokens:
reasoning_tokens = _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
reasoning_rate: Final = _resolve_billed_reasoning_rate(
model_info=model_info,
usage=usage,
@ -1361,23 +1395,9 @@ def get_token_type_cost_breakdown(
completion_base_cost=completion_base_cost,
current_time=billing_time,
)
reasoning_cost = float(reasoning_tokens) * reasoning_rate
cache_read_tokens = 0
cache_creation_tokens = 0
cache_creation_token_details: CacheCreationTokenDetails | None = None
if usage.prompt_tokens_details is not None:
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens = prompt_tokens_details["cache_hit_tokens"]
cache_creation_tokens = prompt_tokens_details["cache_creation_tokens"]
cache_creation_token_details = prompt_tokens_details["cache_creation_token_details"]
# Fall back to the private top-level counters the Usage constructor mirrors cache
# tokens onto, so providers/callers that bypass prompt_tokens_details are covered.
if not cache_read_tokens:
cache_read_tokens = _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0))
if not cache_creation_tokens:
cache_creation_tokens = _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0))
reasoning_cost = float(_reasoning_token_count(usage)) * reasoning_rate
cache_read_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(usage)
cache_read_cost = float(cache_read_tokens) * cache_read_cost_rate
cache_creation_cost = calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,

36
litellm/proxy/_types.py
View file

@ -5137,9 +5137,26 @@ class CostEstimateRequest(LiteLLMPydanticObjectBase):
model: str = Field(description="Model name (from /model_group/info)")
input_tokens: int = Field(description="Expected input tokens per request", ge=0)
output_tokens: int = Field(description="Expected output tokens per request", ge=0)
cache_read_input_tokens: int = Field(
default=0, description="Input tokens read from the prompt cache; counted within input_tokens", ge=0
)
cache_creation_input_tokens: int = Field(
default=0, description="Input tokens written to the prompt cache; counted within input_tokens", ge=0
)
reasoning_tokens: int = Field(
default=0, description="Reasoning tokens the model emits; counted within output_tokens", ge=0
)
num_requests_per_day: int | None = Field(default=None, description="Number of requests per day", ge=0)
num_requests_per_month: int | None = Field(default=None, description="Number of requests per month", ge=0)
@model_validator(mode="after")
def validate_token_subsets(self) -> "CostEstimateRequest":
if self.cache_read_input_tokens + self.cache_creation_input_tokens > self.input_tokens:
raise ValueError("cache_read_input_tokens plus cache_creation_input_tokens cannot exceed input_tokens")
if self.reasoning_tokens > self.output_tokens:
raise ValueError("reasoning_tokens cannot exceed output_tokens")
return self
class CostEstimateResponse(LiteLLMPydanticObjectBase):
"""Response body for /cost/estimate endpoint."""
@ -5147,6 +5164,9 @@ class CostEstimateResponse(LiteLLMPydanticObjectBase):
model: str
input_tokens: int
output_tokens: int
cache_read_input_tokens: int = 0
cache_creation_input_tokens: int = 0
reasoning_tokens: int = 0
num_requests_per_day: int | None = None
num_requests_per_month: int | None = None
# Per-request costs
@ -5154,17 +5174,33 @@ class CostEstimateResponse(LiteLLMPydanticObjectBase):
input_cost_per_request: float = Field(description="Input token cost per request (before margin)")
output_cost_per_request: float = Field(description="Output token cost per request (before margin)")
margin_cost_per_request: float = Field(default=0.0, description="Margin/fee added per request")
cache_read_cost_per_request: float = Field(default=0.0, description="Cache-read share of input_cost_per_request")
cache_creation_cost_per_request: float = Field(
default=0.0, description="Cache-write share of input_cost_per_request"
)
reasoning_cost_per_request: float = Field(default=0.0, description="Reasoning share of output_cost_per_request")
# Daily costs (if num_requests_per_day provided)
daily_cost: float | None = Field(default=None, description="Total daily cost (includes margin)")
daily_input_cost: float | None = Field(default=None, description="Daily input token cost")
daily_output_cost: float | None = Field(default=None, description="Daily output token cost")
daily_margin_cost: float | None = Field(default=None, description="Daily margin/fee")
daily_cache_read_cost: float | None = Field(default=None, description="Cache-read share of daily_input_cost")
daily_cache_creation_cost: float | None = Field(default=None, description="Cache-write share of daily_input_cost")
daily_reasoning_cost: float | None = Field(default=None, description="Reasoning share of daily_output_cost")
# Monthly costs (if num_requests_per_month provided)
monthly_cost: float | None = Field(default=None, description="Total monthly cost (includes margin)")
monthly_input_cost: float | None = Field(default=None, description="Monthly input token cost")
monthly_output_cost: float | None = Field(default=None, description="Monthly output token cost")
monthly_margin_cost: float | None = Field(default=None, description="Monthly margin/fee")
monthly_cache_read_cost: float | None = Field(default=None, description="Cache-read share of monthly_input_cost")
monthly_cache_creation_cost: float | None = Field(
default=None, description="Cache-write share of monthly_input_cost"
)
monthly_reasoning_cost: float | None = Field(default=None, description="Reasoning share of monthly_output_cost")
# Pricing info
input_cost_per_token: float | None = None
output_cost_per_token: float | None = None
cache_read_input_token_cost: float | None = Field(default=None, description="Rate billed per cache-read token")
cache_creation_input_token_cost: float | None = Field(default=None, description="Rate billed per cache-write token")
output_cost_per_reasoning_token: float | None = Field(default=None, description="Rate billed per reasoning token")
provider: str | None = None

253
litellm/proxy/management_endpoints/cost_tracking_settings.py
View file

@ -27,7 +27,15 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.utils import CostPerToken, LlmProvidersSet, ModelInfo
from litellm.types.utils import (
CostBreakdown,
CostPerToken,
LlmProvidersSet,
ModelInfo,
ModelResponse,
PromptTokensDetailsWrapper,
Usage,
)
router: Final = APIRouter()
@ -46,13 +54,15 @@ def _configured_price(key: str, sources: tuple[Mapping[str, object], ...]) -> fl
def _extract_custom_pricing(
litellm_params: Mapping[str, object], model_info: Mapping[str, object]
litellm_params: Mapping[str, object], model_info: Mapping[str, object], builtin: ModelInfo | None
) -> CostPerToken | None:
"""
Pull per-token pricing configured on a deployment so on-prem / self-hosted
models (absent from the public cost map) still estimate a real cost.
Pricing may live on ``litellm_params`` or ``model_info``; ``litellm_params``
wins, matching the router's cost-map registration precedence.
wins, matching the router's cost-map registration precedence. Cache rates the
deployment leaves unset come from the backend model's built-in entry, then its
own input rate, again matching what the router registers for live billing.
"""
sources: Final = (litellm_params, model_info)
input_price: Final = _configured_price("input_cost_per_token", sources)
@ -61,9 +71,15 @@ def _extract_custom_pricing(
if input_price is None and output_price is None:
return None
input_rate: Final = input_price or 0.0
cache_sources: Final = sources if builtin is None else (*sources, builtin)
cache_read_price: Final = _configured_price("cache_read_input_token_cost", cache_sources)
cache_creation_price: Final = _configured_price("cache_creation_input_token_cost", cache_sources)
return CostPerToken(
input_cost_per_token=input_price or 0.0,
input_cost_per_token=input_rate,
output_cost_per_token=output_price or 0.0,
cache_read_input_token_cost=input_rate if cache_read_price is None else cache_read_price,
cache_creation_input_token_cost=input_rate if cache_creation_price is None else cache_creation_price,
)
@ -98,17 +114,14 @@ def _resolve_model_for_cost_lookup(model: str) -> ResolvedCostModel:
model_info: Final = first_deployment.get("model_info", {})
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
provider: Final = str(custom_llm_provider) if custom_llm_provider is not None else None
custom_cost_per_token: Final = _extract_custom_pricing(litellm_params, model_info)
# Check base_model first (needed for Azure custom deployment names)
# base_model wins (needed for Azure custom deployment names)
base_model: Final = model_info.get("base_model") or litellm_params.get("base_model")
if base_model:
verbose_proxy_logger.debug("Resolved model '%s' to base_model '%s' from router", model, base_model)
return ResolvedCostModel(str(base_model), provider, custom_cost_per_token)
resolved_model: Final = litellm_params.get("model")
resolved_model: Final = base_model or litellm_params.get("model")
if resolved_model:
verbose_proxy_logger.debug("Resolved model '%s' to '%s' from router", model, resolved_model)
custom_cost_per_token: Final = _extract_custom_pricing(
litellm_params, model_info, _lookup_model_info(str(resolved_model))
)
return ResolvedCostModel(str(resolved_model), provider, custom_cost_per_token)
except Exception as e:
verbose_proxy_logger.debug("Could not resolve model '%s' from router: %s", model, e)
@ -117,19 +130,97 @@ def _resolve_model_for_cost_lookup(model: str) -> ResolvedCostModel:
return ResolvedCostModel(model, None, None)
def _calculate_period_costs(num_requests, cost_per_request, input_cost, output_cost, margin_cost):
"""
Calculate costs for a given number of requests.
@dataclass(frozen=True, slots=True)
class CostLines:
"""Cost of one request split the way the spend logs split it: the cache lines are
shares of input_cost and the reasoning line is a share of output_cost."""
Returns tuple of (total_cost, input_cost, output_cost, margin_cost) or all None if num_requests is None/0.
"""
if not num_requests:
return None, None, None, None
return (
cost_per_request * num_requests,
input_cost * num_requests,
output_cost * num_requests,
margin_cost * num_requests,
total_cost: float
input_cost: float
output_cost: float
margin_cost: float
cache_read_cost: float
cache_creation_cost: float
reasoning_cost: float
def times(self, num_requests: int | None) -> "CostLines | None":
if not num_requests:
return None
return CostLines(
total_cost=self.total_cost * num_requests,
input_cost=self.input_cost * num_requests,
output_cost=self.output_cost * num_requests,
margin_cost=self.margin_cost * num_requests,
cache_read_cost=self.cache_read_cost * num_requests,
cache_creation_cost=self.cache_creation_cost * num_requests,
reasoning_cost=self.reasoning_cost * num_requests,
)
def _cost_lines(cost_per_request: float, cost_breakdown: CostBreakdown | None) -> CostLines:
breakdown: Final = cost_breakdown if cost_breakdown is not None else CostBreakdown()
return CostLines(
total_cost=cost_per_request,
input_cost=breakdown.get("input_cost", 0.0),
output_cost=breakdown.get("output_cost", 0.0),
margin_cost=breakdown.get("margin_total_amount", 0.0),
cache_read_cost=breakdown.get("cache_read_cost", 0.0),
cache_creation_cost=breakdown.get("cache_creation_cost", 0.0),
reasoning_cost=breakdown.get("reasoning_cost", 0.0),
)
@dataclass(frozen=True, slots=True)
class EffectiveTokenRates:
input_cost_per_token: float | None
output_cost_per_token: float | None
cache_read_input_token_cost: float | None
cache_creation_input_token_cost: float | None
output_cost_per_reasoning_token: float | None
def _custom_token_rates(custom_cost_per_token: CostPerToken) -> EffectiveTokenRates:
input_rate: Final = custom_cost_per_token["input_cost_per_token"]
output_rate: Final = custom_cost_per_token["output_cost_per_token"]
return EffectiveTokenRates(
input_cost_per_token=input_rate,
output_cost_per_token=output_rate,
cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate),
cache_creation_input_token_cost=custom_cost_per_token.get("cache_creation_input_token_cost", input_rate),
output_cost_per_reasoning_token=output_rate,
)
def _cost_map_token_rates(model_info: ModelInfo | None) -> EffectiveTokenRates:
"""Base rates the cost calculator bills flat usage at: a cost-map model without a cache price
bills cache tokens at zero, and one without a reasoning price bills reasoning at the output rate."""
if model_info is None:
return EffectiveTokenRates(None, None, None, None, None)
sources: Final = (model_info,)
output_rate: Final = _configured_price("output_cost_per_token", sources)
reasoning_rate: Final = _configured_price("output_cost_per_reasoning_token", sources)
return EffectiveTokenRates(
input_cost_per_token=_configured_price("input_cost_per_token", sources),
output_cost_per_token=output_rate,
cache_read_input_token_cost=_configured_price("cache_read_input_token_cost", sources) or 0.0,
cache_creation_input_token_cost=_configured_price("cache_creation_input_token_cost", sources) or 0.0,
output_cost_per_reasoning_token=output_rate if reasoning_rate is None else reasoning_rate,
)
def _usage_for_estimate(request: CostEstimateRequest) -> Usage:
cache_tokens: Final = request.cache_read_input_tokens + request.cache_creation_input_tokens
return Usage(
prompt_tokens=request.input_tokens,
completion_tokens=request.output_tokens,
total_tokens=request.input_tokens + request.output_tokens,
reasoning_tokens=request.reasoning_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=request.cache_read_input_tokens,
cache_creation_tokens=request.cache_creation_input_tokens,
)
if cache_tokens
else None,
)
@ -530,11 +621,14 @@ async def estimate_cost(
- model: Model name (e.g., "gpt-4", "claude-3-opus")
- input_tokens: Expected input tokens per request
- output_tokens: Expected output tokens per request
- cache_read_input_tokens: Cache-read tokens per request, counted within input_tokens (optional)
- cache_creation_input_tokens: Cache-write tokens per request, counted within input_tokens (optional)
- reasoning_tokens: Reasoning tokens per request, counted within output_tokens (optional)
- num_requests_per_day: Number of requests per day (optional)
- num_requests_per_month: Number of requests per month (optional)
Returns cost breakdown including:
- Per-request costs (input, output, margin)
- Per-request costs (input, output, margin, plus the cache-read, cache-write and reasoning shares)
- Daily costs (if num_requests_per_day provided)
- Monthly costs (if num_requests_per_month provided)
@ -543,14 +637,15 @@ async def estimate_cost(
{
"model": "gpt-4",
"input_tokens": 1000,
"cache_read_input_tokens": 800,
"output_tokens": 500,
"reasoning_tokens": 200,
"num_requests_per_day": 100,
"num_requests_per_month": 3000
}
```
"""
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import ModelResponse, Usage
# Resolve model name (handles router aliases like 'e-model-router' -> 'azure_ai/gpt-4')
resolved: Final = _resolve_model_for_cost_lookup(request.model)
@ -559,15 +654,7 @@ async def estimate_cost(
verbose_proxy_logger.debug("Cost estimate: request.model='%s' resolved to '%s'", request.model, resolved_model)
# Create a mock response with usage for completion_cost
mock_response: Final = ModelResponse(
model=resolved_model,
usage=Usage(
prompt_tokens=request.input_tokens,
completion_tokens=request.output_tokens,
total_tokens=request.input_tokens + request.output_tokens,
),
)
mock_response: Final = ModelResponse(model=resolved_model, usage=_usage_for_estimate(request))
# Create a logging object to capture cost breakdown
litellm_logging_obj: Final = LiteLLMLoggingObj(
@ -597,75 +684,53 @@ async def estimate_cost(
},
)
# Get cost breakdown from the logging object
cost_breakdown: Final = litellm_logging_obj.cost_breakdown
input_cost: Final = cost_breakdown.get("input_cost", 0.0) if cost_breakdown else 0.0
output_cost: Final = cost_breakdown.get("output_cost", 0.0) if cost_breakdown else 0.0
margin_cost: Final = cost_breakdown.get("margin_total_amount", 0.0) if cost_breakdown else 0.0
per_request: Final = _cost_lines(cost_per_request, litellm_logging_obj.cost_breakdown)
daily: Final = per_request.times(request.num_requests_per_day)
monthly: Final = per_request.times(request.num_requests_per_month)
model_info: Final = _lookup_model_info(resolved_model)
mapped_input_price: Final = model_info.get("input_cost_per_token") if model_info is not None else None
mapped_output_price: Final = model_info.get("output_cost_per_token") if model_info is not None else None
rates: Final = (
_custom_token_rates(resolved.custom_cost_per_token)
if resolved.custom_cost_per_token is not None
else _cost_map_token_rates(model_info)
)
mapped_provider: Final = model_info.get("litellm_provider") if model_info is not None else None
input_cost_per_token: Final = (
resolved.custom_cost_per_token["input_cost_per_token"]
if resolved.custom_cost_per_token is not None
else mapped_input_price
)
output_cost_per_token: Final = (
resolved.custom_cost_per_token["output_cost_per_token"]
if resolved.custom_cost_per_token is not None
else mapped_output_price
)
custom_llm_provider: Final = mapped_provider if mapped_provider is not None else resolved_provider
# Calculate daily and monthly costs
(
daily_cost,
daily_input_cost,
daily_output_cost,
daily_margin_cost,
) = _calculate_period_costs(
num_requests=request.num_requests_per_day,
cost_per_request=cost_per_request,
input_cost=input_cost,
output_cost=output_cost,
margin_cost=margin_cost,
)
(
monthly_cost,
monthly_input_cost,
monthly_output_cost,
monthly_margin_cost,
) = _calculate_period_costs(
num_requests=request.num_requests_per_month,
cost_per_request=cost_per_request,
input_cost=input_cost,
output_cost=output_cost,
margin_cost=margin_cost,
)
return CostEstimateResponse(
model=request.model,
input_tokens=request.input_tokens,
output_tokens=request.output_tokens,
cache_read_input_tokens=request.cache_read_input_tokens,
cache_creation_input_tokens=request.cache_creation_input_tokens,
reasoning_tokens=request.reasoning_tokens,
num_requests_per_day=request.num_requests_per_day,
num_requests_per_month=request.num_requests_per_month,
cost_per_request=cost_per_request,
input_cost_per_request=input_cost,
output_cost_per_request=output_cost,
margin_cost_per_request=margin_cost,
daily_cost=daily_cost,
daily_input_cost=daily_input_cost,
daily_output_cost=daily_output_cost,
daily_margin_cost=daily_margin_cost,
monthly_cost=monthly_cost,
monthly_input_cost=monthly_input_cost,
monthly_output_cost=monthly_output_cost,
monthly_margin_cost=monthly_margin_cost,
input_cost_per_token=input_cost_per_token,
output_cost_per_token=output_cost_per_token,
cost_per_request=per_request.total_cost,
input_cost_per_request=per_request.input_cost,
output_cost_per_request=per_request.output_cost,
margin_cost_per_request=per_request.margin_cost,
cache_read_cost_per_request=per_request.cache_read_cost,
cache_creation_cost_per_request=per_request.cache_creation_cost,
reasoning_cost_per_request=per_request.reasoning_cost,
daily_cost=daily.total_cost if daily is not None else None,
daily_input_cost=daily.input_cost if daily is not None else None,
daily_output_cost=daily.output_cost if daily is not None else None,
daily_margin_cost=daily.margin_cost if daily is not None else None,
daily_cache_read_cost=daily.cache_read_cost if daily is not None else None,
daily_cache_creation_cost=daily.cache_creation_cost if daily is not None else None,
daily_reasoning_cost=daily.reasoning_cost if daily is not None else None,
monthly_cost=monthly.total_cost if monthly is not None else None,
monthly_input_cost=monthly.input_cost if monthly is not None else None,
monthly_output_cost=monthly.output_cost if monthly is not None else None,
monthly_margin_cost=monthly.margin_cost if monthly is not None else None,
monthly_cache_read_cost=monthly.cache_read_cost if monthly is not None else None,
monthly_cache_creation_cost=monthly.cache_creation_cost if monthly is not None else None,
monthly_reasoning_cost=monthly.reasoning_cost if monthly is not None else None,
input_cost_per_token=rates.input_cost_per_token,
output_cost_per_token=rates.output_cost_per_token,
cache_read_input_token_cost=rates.cache_read_input_token_cost,
cache_creation_input_token_cost=rates.cache_creation_input_token_cost,
output_cost_per_reasoning_token=rates.output_cost_per_reasoning_token,
provider=custom_llm_provider,
)

64
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -3874,6 +3874,70 @@ def test_token_type_cost_breakdown_reconciles_with_generic_total(_local_model_co
assert text_input_cost + breakdown.cache_read_cost == pytest.approx(prompt_cost)
def _custom_priced_usage() -> Usage:
return Usage(
prompt_tokens=1000,
completion_tokens=500,
total_tokens=1500,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800, cache_creation_tokens=100),
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200),
)
def test_token_type_cost_breakdown_prices_custom_pricing_from_its_flat_rates():
"""
A custom-priced deployment, usually absent from the cost map, used to get zero cache and
reasoning lines while its total already billed cache tokens at the custom cache rates.
The lines must come from the same flat rates: a configured cache rate, else the input
rate for cache tokens and the output rate for reasoning tokens.
"""
from litellm.types.utils import CostPerToken
breakdown = get_token_type_cost_breakdown(
model="openai/onprem-model",
custom_llm_provider="openai",
usage=_custom_priced_usage(),
custom_cost_per_token=CostPerToken(
input_cost_per_token=1e-6, output_cost_per_token=2e-6, cache_read_input_token_cost=1e-7
),
)
assert breakdown.cache_read_cost == pytest.approx(800 * 1e-7)
assert breakdown.cache_creation_cost == pytest.approx(100 * 1e-6)
assert breakdown.reasoning_cost == pytest.approx(200 * 2e-6)
def test_token_type_cost_breakdown_reconciles_with_custom_pricing_totals():
from litellm.cost_calculator import cost_per_token
from litellm.types.utils import CostPerToken
usage = _custom_priced_usage()
custom_cost_per_token = CostPerToken(
input_cost_per_token=1e-6,
output_cost_per_token=2e-6,
cache_read_input_token_cost=1e-7,
cache_creation_input_token_cost=1.25e-6,
)
prompt_cost, completion_cost = cost_per_token(
model="openai/onprem-model",
custom_llm_provider="openai",
prompt_tokens=1000,
completion_tokens=500,
usage_object=usage,
custom_cost_per_token=custom_cost_per_token,
)
breakdown = get_token_type_cost_breakdown(
model="openai/onprem-model",
custom_llm_provider="openai",
usage=usage,
custom_cost_per_token=custom_cost_per_token,
)
assert 100 * 1e-6 + breakdown.cache_read_cost + breakdown.cache_creation_cost == pytest.approx(prompt_cost)
assert 300 * 2e-6 + breakdown.reasoning_cost == pytest.approx(completion_cost)
def test_token_type_cost_breakdown_zero_without_special_tokens(_local_model_cost_map):
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)

187
tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py
View file

@ -8,9 +8,11 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from pydantic import ValidationError
import litellm
from litellm.proxy._types import CostEstimateRequest
from litellm.proxy.management_endpoints.cost_tracking_settings import router
from litellm.proxy.proxy_server import app
@ -789,13 +791,13 @@ INPUT_TOKENS = 1000
OUTPUT_TOKENS = 500
def _router_pricing(**pricing: float) -> MagicMock:
def _router_pricing(model: str = AN_UNDERLYING_MODEL, **pricing: float) -> MagicMock:
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
{
"model_name": AN_ALIAS,
"litellm_params": {
"model": AN_UNDERLYING_MODEL,
"model": model,
"custom_llm_provider": "openai",
**pricing,
},
@ -909,3 +911,184 @@ class TestEstimateCostPeriodTotals:
assert response.cost_per_request == pytest.approx(0.0022)
assert response.daily_margin_cost == pytest.approx(0.02)
assert response.daily_cost == pytest.approx(0.22)
CACHE_READ_TOKENS = 800
CACHE_CREATION_TOKENS = 100
REASONING_TOKENS = 200
TEXT_INPUT_TOKENS = INPUT_TOKENS - CACHE_READ_TOKENS - CACHE_CREATION_TOKENS
TEXT_OUTPUT_TOKENS = OUTPUT_TOKENS - REASONING_TOKENS
async def _estimate_with_cache_and_reasoning(mock_router: MagicMock | None, model: str = AN_ALIAS, **overrides: int):
return await _estimate(
mock_router,
model=model,
cache_read_input_tokens=CACHE_READ_TOKENS,
cache_creation_input_tokens=CACHE_CREATION_TOKENS,
reasoning_tokens=REASONING_TOKENS,
**overrides,
)
class TestEstimateCostCacheAndReasoningTokens:
@pytest.mark.asyncio
async def test_a_mapped_model_bills_cache_and_reasoning_tokens_at_their_own_rates(self, monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"cache_read_input_token_cost": 3e-7,
"cache_creation_input_token_cost": 3.75e-6,
"output_cost_per_reasoning_token": 1e-5,
"litellm_provider": "openai",
"mode": "chat",
},
)
response = await _estimate_with_cache_and_reasoning(None, model=A_MAPPED_MODEL, num_requests_per_day=10)
assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 3e-7)
assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 3.75e-6)
assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 1e-5)
assert response.input_cost_per_request == pytest.approx(
TEXT_INPUT_TOKENS * 3e-6 + CACHE_READ_TOKENS * 3e-7 + CACHE_CREATION_TOKENS * 3.75e-6
)
assert response.output_cost_per_request == pytest.approx(TEXT_OUTPUT_TOKENS * 15e-6 + REASONING_TOKENS * 1e-5)
assert response.cost_per_request == pytest.approx(
response.input_cost_per_request + response.output_cost_per_request
)
assert response.daily_cache_read_cost == pytest.approx(10 * CACHE_READ_TOKENS * 3e-7)
assert response.daily_cache_creation_cost == pytest.approx(10 * CACHE_CREATION_TOKENS * 3.75e-6)
assert response.daily_reasoning_cost == pytest.approx(10 * REASONING_TOKENS * 1e-5)
assert response.monthly_cache_read_cost is None
assert response.cache_read_input_token_cost == pytest.approx(3e-7)
assert response.cache_creation_input_token_cost == pytest.approx(3.75e-6)
assert response.output_cost_per_reasoning_token == pytest.approx(1e-5)
assert (
response.cache_read_input_tokens,
response.cache_creation_input_tokens,
response.reasoning_tokens,
) == (CACHE_READ_TOKENS, CACHE_CREATION_TOKENS, REASONING_TOKENS)
@pytest.mark.asyncio
async def test_a_model_without_cache_or_reasoning_prices_estimates_what_the_proxy_bills(self, monkeypatch):
"""The cost calculator bills cache tokens of a cost-map model without cache prices at zero
and its reasoning tokens at the output rate. The estimate reports those effective rates."""
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{"input_cost_per_token": 5e-6, "output_cost_per_token": 6e-6, "litellm_provider": "openai", "mode": "chat"},
)
response = await _estimate_with_cache_and_reasoning(None, model=A_MAPPED_MODEL)
assert response.cache_read_cost_per_request == 0.0
assert response.cache_creation_cost_per_request == 0.0
assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 6e-6)
assert response.input_cost_per_request == pytest.approx(TEXT_INPUT_TOKENS * 5e-6)
assert response.cost_per_request == pytest.approx(TEXT_INPUT_TOKENS * 5e-6 + OUTPUT_TOKENS * 6e-6)
assert response.cache_read_input_token_cost == 0.0
assert response.cache_creation_input_token_cost == 0.0
assert response.output_cost_per_reasoning_token == pytest.approx(6e-6)
@pytest.mark.asyncio
async def test_a_request_without_cache_or_reasoning_tokens_estimates_as_before(self, monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"cache_read_input_token_cost": 3e-7,
"cache_creation_input_token_cost": 3.75e-6,
"output_cost_per_reasoning_token": 1e-5,
"litellm_provider": "openai",
"mode": "chat",
},
)
response = await _estimate(None, model=A_MAPPED_MODEL, num_requests_per_day=10)
assert response.cost_per_request == pytest.approx(INPUT_TOKENS * 3e-6 + OUTPUT_TOKENS * 15e-6)
assert response.cache_read_cost_per_request == 0.0
assert response.cache_creation_cost_per_request == 0.0
assert response.reasoning_cost_per_request == 0.0
assert response.daily_cache_read_cost == 0.0
assert response.daily_reasoning_cost == 0.0
@pytest.mark.asyncio
async def test_a_custom_priced_deployment_bills_cache_and_reasoning_tokens_from_its_flat_rates(self):
response = await _estimate_with_cache_and_reasoning(
_router_pricing(input_cost_per_token=1e-6, output_cost_per_token=2e-6, cache_read_input_token_cost=1e-7)
)
assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 1e-7)
assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 1e-6)
assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 2e-6)
assert response.cost_per_request == pytest.approx(
TEXT_INPUT_TOKENS * 1e-6 + CACHE_READ_TOKENS * 1e-7 + CACHE_CREATION_TOKENS * 1e-6 + OUTPUT_TOKENS * 2e-6
)
assert response.cache_read_input_token_cost == pytest.approx(1e-7)
assert response.cache_creation_input_token_cost == pytest.approx(1e-6)
assert response.output_cost_per_reasoning_token == pytest.approx(2e-6)
@pytest.mark.asyncio
async def test_a_custom_priced_deployment_of_a_mapped_model_inherits_its_built_in_cache_rates(self, monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{
"input_cost_per_token": 5e-6,
"output_cost_per_token": 6e-6,
"cache_read_input_token_cost": 5e-7,
"cache_creation_input_token_cost": 6.25e-6,
"litellm_provider": "openai",
"mode": "chat",
},
)
response = await _estimate_with_cache_and_reasoning(
_router_pricing(model=A_MAPPED_MODEL, input_cost_per_token=1e-6, output_cost_per_token=2e-6)
)
assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 5e-7)
assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 6.25e-6)
assert response.input_cost_per_request == pytest.approx(
TEXT_INPUT_TOKENS * 1e-6 + CACHE_READ_TOKENS * 5e-7 + CACHE_CREATION_TOKENS * 6.25e-6
)
assert response.cache_read_input_token_cost == pytest.approx(5e-7)
assert response.cache_creation_input_token_cost == pytest.approx(6.25e-6)
class TestCostEstimateRequestTokenSubsets:
def test_cache_tokens_beyond_the_input_tokens_are_rejected(self):
with pytest.raises(ValidationError, match="cannot exceed input_tokens"):
CostEstimateRequest(
model=AN_ALIAS,
input_tokens=INPUT_TOKENS,
output_tokens=OUTPUT_TOKENS,
cache_read_input_tokens=INPUT_TOKENS,
cache_creation_input_tokens=1,
)
def test_reasoning_tokens_beyond_the_output_tokens_are_rejected(self):
with pytest.raises(ValidationError, match="cannot exceed output_tokens"):
CostEstimateRequest(
model=AN_ALIAS,
input_tokens=INPUT_TOKENS,
output_tokens=OUTPUT_TOKENS,
reasoning_tokens=OUTPUT_TOKENS + 1,
)
def test_the_endpoint_answers_422_when_cache_tokens_exceed_input_tokens(self):
response = client.post(
"/cost/estimate",
headers={"Authorization": "Bearer sk-1234"},
json={"model": AN_ALIAS, "input_tokens": 1000, "output_tokens": 100, "cache_read_input_tokens": 8000},
)
assert response.status_code == 422
assert "cannot exceed input_tokens" in response.text

50
tests/test_litellm/test_cost_calculator.py
View file

@ -3736,6 +3736,56 @@ def test_completion_cost_logs_reasoning_and_cache_breakdown(_local_model_cost_ma
assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(100 * 3e-08)
def test_completion_cost_logs_cache_and_reasoning_breakdown_for_custom_pricing():
"""
A custom-priced deployment bills cache tokens at its custom cache rates, but the
breakdown stored for the spend logs carried no cache or reasoning lines for it.
"""
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.utils import CompletionTokensDetailsWrapper, CostPerToken
logging_obj = Logging(
model="openai/onprem-model",
messages=[{"role": "user", "content": "Hello"}],
stream=False,
call_type="completion",
start_time=datetime.now(),
litellm_call_id="custom-pricing-breakdown",
function_id="f",
)
response = ModelResponse(
model="openai/onprem-model",
usage=Usage(
prompt_tokens=1000,
completion_tokens=500,
total_tokens=1500,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800, cache_creation_tokens=100),
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200),
),
)
total = completion_cost(
completion_response=response,
model="openai/onprem-model",
custom_llm_provider="openai",
custom_cost_per_token=CostPerToken(
input_cost_per_token=1e-6,
output_cost_per_token=2e-6,
cache_read_input_token_cost=1e-7,
cache_creation_input_token_cost=1.25e-6,
),
litellm_logging_obj=logging_obj,
)
assert logging_obj.cost_breakdown is not None
assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(800 * 1e-7)
assert logging_obj.cost_breakdown["cache_creation_cost"] == pytest.approx(100 * 1.25e-6)
assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(200 * 2e-6)
assert total == pytest.approx(100 * 1e-6 + 800 * 1e-7 + 100 * 1.25e-6 + 500 * 2e-6)
def test_cost_per_token_per_second_pricing(monkeypatch):
"""
Models priced by duration (input/output_cost_per_second) with no per-token rates

103
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -3301,11 +3301,14 @@ export interface paths {
* - model: Model name (e.g., "gpt-4", "claude-3-opus")
* - input_tokens: Expected input tokens per request
* - output_tokens: Expected output tokens per request
* - cache_read_input_tokens: Cache-read tokens per request, counted within input_tokens (optional)
* - cache_creation_input_tokens: Cache-write tokens per request, counted within input_tokens (optional)
* - reasoning_tokens: Reasoning tokens per request, counted within output_tokens (optional)
* - num_requests_per_day: Number of requests per day (optional)
* - num_requests_per_month: Number of requests per month (optional)
*
* Returns cost breakdown including:
* - Per-request costs (input, output, margin)
* - Per-request costs (input, output, margin, plus the cache-read, cache-write and reasoning shares)
* - Daily costs (if num_requests_per_day provided)
* - Monthly costs (if num_requests_per_month provided)
*
@ -3314,7 +3317,9 @@ export interface paths {
* {
* "model": "gpt-4",
* "input_tokens": 1000,
* "cache_read_input_tokens": 800,
* "output_tokens": 500,
* "reasoning_tokens": 200,
* "num_requests_per_day": 100,
* "num_requests_per_month": 3000
* }
@ -26392,6 +26397,18 @@ export interface components {
* @description Request body for /cost/estimate endpoint.
*/
CostEstimateRequest: {
/**
* Cache Creation Input Tokens
* @description Input tokens written to the prompt cache; counted within input_tokens
* @default 0
*/
cache_creation_input_tokens: number;
/**
* Cache Read Input Tokens
* @description Input tokens read from the prompt cache; counted within input_tokens
* @default 0
*/
cache_read_input_tokens: number;
/**
* Input Tokens
* @description Expected input tokens per request
@ -26417,17 +26434,65 @@ export interface components {
* @description Expected output tokens per request
*/
output_tokens: number;
/**
* Reasoning Tokens
* @description Reasoning tokens the model emits; counted within output_tokens
* @default 0
*/
reasoning_tokens: number;
};
/**
* CostEstimateResponse
* @description Response body for /cost/estimate endpoint.
*/
CostEstimateResponse: {
/**
* Cache Creation Cost Per Request
* @description Cache-write share of input_cost_per_request
* @default 0
*/
cache_creation_cost_per_request: number;
/**
* Cache Creation Input Token Cost
* @description Rate billed per cache-write token
*/
cache_creation_input_token_cost?: number | null;
/**
* Cache Creation Input Tokens
* @default 0
*/
cache_creation_input_tokens: number;
/**
* Cache Read Cost Per Request
* @description Cache-read share of input_cost_per_request
* @default 0
*/
cache_read_cost_per_request: number;
/**
* Cache Read Input Token Cost
* @description Rate billed per cache-read token
*/
cache_read_input_token_cost?: number | null;
/**
* Cache Read Input Tokens
* @default 0
*/
cache_read_input_tokens: number;
/**
* Cost Per Request
* @description Total cost per request (includes margin)
*/
cost_per_request: number;
/**
* Daily Cache Creation Cost
* @description Cache-write share of daily_input_cost
*/
daily_cache_creation_cost?: number | null;
/**
* Daily Cache Read Cost
* @description Cache-read share of daily_input_cost
*/
daily_cache_read_cost?: number | null;
/**
* Daily Cost
* @description Total daily cost (includes margin)
@ -26448,6 +26513,11 @@ export interface components {
* @description Daily output token cost
*/
daily_output_cost?: number | null;
/**
* Daily Reasoning Cost
* @description Reasoning share of daily_output_cost
*/
daily_reasoning_cost?: number | null;
/**
* Input Cost Per Request
* @description Input token cost per request (before margin)
@ -26465,6 +26535,16 @@ export interface components {
margin_cost_per_request: number;
/** Model */
model: string;
/**
* Monthly Cache Creation Cost
* @description Cache-write share of monthly_input_cost
*/
monthly_cache_creation_cost?: number | null;
/**
* Monthly Cache Read Cost
* @description Cache-read share of monthly_input_cost
*/
monthly_cache_read_cost?: number | null;
/**
* Monthly Cost
* @description Total monthly cost (includes margin)
@ -26485,10 +26565,20 @@ export interface components {
* @description Monthly output token cost
*/
monthly_output_cost?: number | null;
/**
* Monthly Reasoning Cost
* @description Reasoning share of monthly_output_cost
*/
monthly_reasoning_cost?: number | null;
/** Num Requests Per Day */
num_requests_per_day?: number | null;
/** Num Requests Per Month */
num_requests_per_month?: number | null;
/**
* Output Cost Per Reasoning Token
* @description Rate billed per reasoning token
*/
output_cost_per_reasoning_token?: number | null;
/**
* Output Cost Per Request
* @description Output token cost per request (before margin)
@ -26500,6 +26590,17 @@ export interface components {
output_tokens: number;
/** Provider */
provider?: string | null;
/**
* Reasoning Cost Per Request
* @description Reasoning share of output_cost_per_request
* @default 0
*/
reasoning_cost_per_request: number;
/**
* Reasoning Tokens
* @default 0
*/
reasoning_tokens: number;
};
/** CreateCredentialItem */
CreateCredentialItem: {