scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

Merge pull request #40174 from BerriAI/litellm_cost_estimate_cache_tokens

Browse source 
feat(proxy): price cache and reasoning tokens in /cost/estimate
This commit is contained in:
ryan-crabbe-berri 2026-09-09 10:18:59 -07:00 • committed by GitHub
commit ff2f122846
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 1107 additions and 210 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
litellm/_internal_context.py
View file

@ -6,9 +6,33 @@ be settable from user input. Context variables are scoped to the current
asyncio task and cannot be injected via HTTP request bodies.
"""
from collections.abc import Generator
from contextlib import contextmanager
from contextvars import ContextVar
from datetime import datetime, timezone
from typing import Final
# When True, suppresses async logging and billing for internal sub-calls
# (e.g., emulated file-search steps that make nested LLM calls).
is_internal_call: Final[ContextVar[bool]] = ContextVar("is_internal_call", default=False)
# One request prices its totals, its per-token-type lines and the rates it reports on
# separate code paths. Each reads the clock for off-peak pricing, so without a pinned
# moment they can land on either side of a window boundary and disagree with each other.
_billing_time: Final[ContextVar[datetime | None]] = ContextVar("billing_time", default=None)
@contextmanager
def pinned_billing_time(moment: datetime) -> Generator[None]:
"""Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read."""
token: Final = _billing_time.set(moment)
try:
yield
finally:
_billing_time.reset(token)
def current_billing_time() -> datetime:
"""The pinned billing moment, or now in UTC outside a pinned block."""
pinned: Final = _billing_time.get()
return pinned if pinned is not None else datetime.now(timezone.utc)

7
litellm/cost_calculator.py
View file

@ -25,6 +25,7 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import
TranscriptionUsageObjectTransformation,
)
from litellm.litellm_core_utils.llm_cost_calc.utils import (
BilledTokenRates,
CostCalculatorUtils,
_generic_cost_per_character,
_get_regional_uplift_multiplier,
@ -1125,6 +1126,7 @@ def _store_cost_breakdown_in_logging_obj(
service_tier: str | None = None,
data_residency: str | None = None,
vertex_location: str | None = None,
billed_token_rates: BilledTokenRates | None = None,
) -> None:
"""
Helper function to store cost breakdown in the logging object.
@ -1169,6 +1171,7 @@ def _store_cost_breakdown_in_logging_obj(
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
billed_token_rates=billed_token_rates,
)
except Exception as breakdown_error:
@ -1737,6 +1740,7 @@ def completion_cost(
_reasoning_cost: float | None = None
_cache_read_cost: float | None = None
_cache_creation_cost: float | None = None
_billed_token_rates: BilledTokenRates | None = None
if cost_per_token_usage_object is not None and model:
_breakdown_provider: str | None = (
custom_llm_provider if isinstance(custom_llm_provider, str) else None
@ -1748,10 +1752,12 @@ def completion_cost(
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
custom_cost_per_token=custom_cost_per_token,
)
_reasoning_cost = _token_type_breakdown.reasoning_cost
_cache_read_cost = _token_type_breakdown.cache_read_cost
_cache_creation_cost = _token_type_breakdown.cache_creation_cost
_billed_token_rates = _token_type_breakdown.rates
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar,
@ -1771,6 +1777,7 @@ def completion_cost(
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
billed_token_rates=_billed_token_rates,
)
return _final_cost

5
litellm/litellm_core_utils/litellm_logging.py
View file

@ -203,6 +203,7 @@ if TYPE_CHECKING:
from litellm.integrations.otel.logger import OpenTelemetryV2
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.litellm_core_utils.llm_cost_calc.utils import BilledTokenRates
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
try:
from litellm_enterprise.enterprise_callbacks.callback_controls import (
@ -590,6 +591,7 @@ class Logging(LiteLLMLoggingBaseClass):
# Initialize cost breakdown field
self.cost_breakdown: CostBreakdown | None = None
self.billed_token_rates: BilledTokenRates | None = None
# Init Caching related details
self.caching_details: CachingDetails | None = None
@ -1587,6 +1589,7 @@ class Logging(LiteLLMLoggingBaseClass):
service_tier: str | None = None,
data_residency: str | None = None,
vertex_location: str | None = None,
billed_token_rates: "BilledTokenRates | None" = None,
) -> None:
"""
Helper method to store cost breakdown in the logging object.
@ -1606,8 +1609,10 @@ class Logging(LiteLLMLoggingBaseClass):
service_tier: Tier the costs above were priced on, already resolved
data_residency: Region uplift the costs above were priced on, already resolved
vertex_location: Vertex AI location the costs above were priced on, already resolved
billed_token_rates: Per-token rates the costs above were billed at, already resolved
"""
self.billed_token_rates = billed_token_rates
self.cost_breakdown = CostBreakdown(
input_cost=input_cost,
output_cost=output_cost,

247
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -10,6 +10,7 @@ from typing import Any, Final, Literal, TypedDict, cast
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
import litellm
from litellm._internal_context import current_billing_time
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import (
select_tier_for_input,
@ -19,6 +20,7 @@ from litellm.types.utils import (
CacheCreationTokenDetails,
CallTypes,
CompletionTokensDetailsWrapper,
CostPerToken,
DataResidency,
ImageResponse,
ModelInfo,
@ -305,7 +307,7 @@ def _is_within_off_peak_window(off_peak_hours_utc: str | Sequence[str], current_
than being localised, so callers must pass datetime.now(timezone.utc), never datetime.now(),
or every window shifts by the host's offset.
"""
reference: Final = current_time if current_time is not None else datetime.now(timezone.utc)
reference: Final = current_time if current_time is not None else current_billing_time()
now: Final = (reference.astimezone(timezone.utc) if reference.tzinfo is not None else reference).time()
windows: Final = (off_peak_hours_utc,) if isinstance(off_peak_hours_utc, str) else off_peak_hours_utc
for window in windows:
@ -392,7 +394,7 @@ def _is_off_peak(off_peak: Mapping[str, object], current_time: datetime | None =
rules: the flat hours_utc windows, which apply every day, or any entry in windows, whose
hours apply only on its weekdays.
"""
reference: Final = current_time if current_time is not None else datetime.now(timezone.utc)
reference: Final = current_time if current_time is not None else current_billing_time()
reference_utc: Final = (
reference.astimezone(timezone.utc) if reference.tzinfo is not None else reference.replace(tzinfo=timezone.utc)
)
@ -1195,7 +1197,7 @@ def generic_cost_per_token(
usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0
)
billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc)
billing_time: Final = current_time if current_time is not None else current_billing_time()
(
prompt_base_cost,
completion_base_cost,
@ -1309,42 +1311,90 @@ def _coerce_token_count(value: object) -> int:
return value if isinstance(value, int) and value > 0 else 0
@dataclass(frozen=True, slots=True)
class BilledTokenRates:
"""Per-token rates one request's usage bills at, after token tiers, off-peak windows and the
regional multipliers the totals apply, so each cost line equals its token count times its rate."""
input_cost_per_token: float
output_cost_per_token: float
cache_read_input_token_cost: float
cache_creation_input_token_cost: float
cache_creation_input_token_cost_above_1hr: float
output_cost_per_reasoning_token: float
def scaled(self, multiplier: float) -> "BilledTokenRates":
if multiplier == 1.0:
return self
return BilledTokenRates(
input_cost_per_token=self.input_cost_per_token * multiplier,
output_cost_per_token=self.output_cost_per_token * multiplier,
cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier,
cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier,
cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier,
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
)
@dataclass(frozen=True, slots=True)
class TokenTypeCostBreakdown:
reasoning_cost: float
cache_read_cost: float
cache_creation_cost: float
rates: BilledTokenRates | None = None
"""Rates these lines were billed at, so a caller reporting both cannot resolve them a second,
differently-argued way. None when the model's pricing could not be resolved."""
def get_token_type_cost_breakdown(
model: str,
custom_llm_provider: str | None,
def _reasoning_token_count(usage: Usage) -> int:
parsed: Final = (
parse_completion_tokens_details(usage)["reasoning_tokens"] if usage.completion_tokens_details is not None else 0
)
return parsed or _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetails | None]:
"""(cache read tokens, cache creation tokens, cache creation details): read from prompt_tokens_details
first, then the private top-level counters the Usage constructor mirrors cache tokens onto for
providers/callers that bypass the details."""
parsed: Final = parse_prompt_tokens_details(usage) if usage.prompt_tokens_details is not None else None
parsed_read: Final = parsed["cache_hit_tokens"] if parsed is not None else 0
parsed_creation: Final = parsed["cache_creation_tokens"] if parsed is not None else 0
return (
parsed_read or _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0)),
parsed_creation or _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0)),
parsed["cache_creation_token_details"] if parsed is not None else None,
)
def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRates:
"""Flat custom pricing has no tiers, uplifts or reasoning rate: cache tokens bill at the configured
cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does."""
input_rate: Final = custom_cost_per_token["input_cost_per_token"]
output_rate: Final = custom_cost_per_token["output_cost_per_token"]
cache_creation_rate: Final = custom_cost_per_token.get("cache_creation_input_token_cost", input_rate)
return BilledTokenRates(
input_cost_per_token=input_rate,
output_cost_per_token=output_rate,
cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate),
cache_creation_input_token_cost=cache_creation_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_rate,
output_cost_per_reasoning_token=output_rate,
)
def _cost_map_billed_rates(
model_info: ModelInfo,
usage: Usage,
service_tier: str | None = None,
data_residency: str | None = None,
vertex_location: str | None = None,
current_time: datetime | None = None,
) -> TokenTypeCostBreakdown:
"""
Provider-agnostic cost of reasoning and cache tokens, derived from the usage
object and model pricing alone.
This works for every provider, including Perplexity/Cerebras/Dashscope whose
cost calculators bypass ``generic_cost_per_token``, because cache tokens always
land on ``prompt_tokens_details`` (via the Usage constructor and provider
transformations) and reasoning tokens on ``completion_tokens_details``. It reuses
the same rate-resolution primitives as the total-cost path so the breakdown can
never drift from the totals. Returns zeros (never raises) when the model or its
pricing cannot be resolved.
"""
try:
model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception:
return TokenTypeCostBreakdown(0.0, 0.0, 0.0)
billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc)
custom_llm_provider: str | None,
service_tier: str | None,
data_residency: str | None,
vertex_location: str | None,
current_time: datetime | None,
) -> BilledTokenRates:
billing_time: Final = current_time if current_time is not None else current_billing_time()
(
_prompt_base_cost,
prompt_base_cost,
completion_base_cost,
cache_creation_cost_rate,
cache_creation_cost_above_1hr_rate,
@ -1356,13 +1406,6 @@ def get_token_type_cost_breakdown(
current_time=billing_time,
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
)
reasoning_tokens = (
parse_completion_tokens_details(usage)["reasoning_tokens"] if usage.completion_tokens_details is not None else 0
)
if not reasoning_tokens:
reasoning_tokens = _coerce_token_count(getattr(usage, "reasoning_tokens", 0))
reasoning_rate: Final = _resolve_billed_reasoning_rate(
model_info=model_info,
usage=usage,
@ -1370,57 +1413,103 @@ def get_token_type_cost_breakdown(
completion_base_cost=completion_base_cost,
current_time=billing_time,
)
reasoning_cost = float(reasoning_tokens) * reasoning_rate
multiplier: Final = (
_get_regional_uplift_multiplier(model_info, data_residency)
* get_vertex_regional_endpoint_uplift(model_info, vertex_location)
* get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
)
return BilledTokenRates(
input_cost_per_token=prompt_base_cost,
output_cost_per_token=completion_base_cost,
cache_read_input_token_cost=cache_read_cost_rate,
cache_creation_input_token_cost=cache_creation_cost_rate,
cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
output_cost_per_reasoning_token=reasoning_rate,
).scaled(multiplier)
cache_read_tokens = 0
cache_creation_tokens = 0
cache_creation_token_details: CacheCreationTokenDetails | None = None
if usage.prompt_tokens_details is not None:
prompt_tokens_details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens = prompt_tokens_details["cache_hit_tokens"]
cache_creation_tokens = prompt_tokens_details["cache_creation_tokens"]
cache_creation_token_details = prompt_tokens_details["cache_creation_token_details"]
# Fall back to the private top-level counters the Usage constructor mirrors cache
# tokens onto, so providers/callers that bypass prompt_tokens_details are covered.
if not cache_read_tokens:
cache_read_tokens = _coerce_token_count(getattr(usage, "_cache_read_input_tokens", 0))
if not cache_creation_tokens:
cache_creation_tokens = _coerce_token_count(getattr(usage, "_cache_creation_input_tokens", 0))
cache_read_cost = float(cache_read_tokens) * cache_read_cost_rate
cache_creation_cost = calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr_rate,
cache_creation_cost=cache_creation_cost_rate,
def get_billed_token_rates(
model: str,
custom_llm_provider: str | None,
usage: Usage,
service_tier: str | None = None,
data_residency: str | None = None,
vertex_location: str | None = None,
current_time: datetime | None = None,
custom_cost_per_token: CostPerToken | None = None,
) -> BilledTokenRates | None:
"""Rates the cost calculator bills ``usage`` at, resolved exactly as the totals and the token-type
breakdown resolve them. None when the model's pricing cannot be resolved."""
if custom_cost_per_token is not None:
return _custom_pricing_rates(custom_cost_per_token)
try:
model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception:
return None
return _cost_map_billed_rates(
model_info=model_info,
usage=usage,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
current_time=current_time,
)
# Apply the same flat regional-processing uplift the totals get, so per-type
# costs stay reconciled with input_cost/output_cost for regionalized OpenAI hosts.
uplift: Final = _get_regional_uplift_multiplier(model_info, data_residency)
if uplift != 1.0:
reasoning_cost *= uplift
cache_read_cost *= uplift
cache_creation_cost *= uplift
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
if vertex_uplift != 1.0:
reasoning_cost *= vertex_uplift
cache_read_cost *= vertex_uplift
cache_creation_cost *= vertex_uplift
def get_token_type_cost_breakdown(
model: str,
custom_llm_provider: str | None,
usage: Usage,
service_tier: str | None = None,
data_residency: str | None = None,
vertex_location: str | None = None,
current_time: datetime | None = None,
custom_cost_per_token: CostPerToken | None = None,
) -> TokenTypeCostBreakdown:
"""
Provider-agnostic cost of reasoning and cache tokens, derived from the usage
object and model pricing alone.
# Mirror the provider-specific geo uplift (e.g. Anthropic us: 1.1) the totals
# apply, so cache and reasoning line items stay reconciled with them.
geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
if geo_multiplier != 1.0:
reasoning_cost *= geo_multiplier
cache_read_cost *= geo_multiplier
cache_creation_cost *= geo_multiplier
This works for every provider, including Perplexity/Cerebras/Dashscope whose
cost calculators bypass ``generic_cost_per_token``, because cache tokens always
land on ``prompt_tokens_details`` (via the Usage constructor and provider
transformations) and reasoning tokens on ``completion_tokens_details``. It reuses
the same rate resolution as the total-cost path (``get_billed_token_rates``) so the
breakdown can never drift from the totals. A deployment billed by
``custom_cost_per_token`` is priced from those flat rates instead of the cost map and,
like its totals, bills cache writes flat rather than by their 5m/1h split.
Returns zeros (never raises) when the model or its pricing cannot be resolved.
"""
rates: Final = get_billed_token_rates(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
current_time=current_time,
custom_cost_per_token=custom_cost_per_token,
)
if rates is None:
return TokenTypeCostBreakdown(0.0, 0.0, 0.0)
cache_read_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(usage)
cache_creation_cost: Final = (
float(cache_creation_tokens) * rates.cache_creation_input_token_cost
if custom_cost_per_token is not None
else calculate_cache_writing_cost(
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
cache_creation_cost_above_1hr=rates.cache_creation_input_token_cost_above_1hr,
cache_creation_cost=rates.cache_creation_input_token_cost,
)
)
return TokenTypeCostBreakdown(
reasoning_cost=reasoning_cost,
cache_read_cost=cache_read_cost,
reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token,
cache_read_cost=float(cache_read_tokens) * rates.cache_read_input_token_cost,
cache_creation_cost=cache_creation_cost,
rates=rates,
)

42
litellm/proxy/_types.py
View file

@ -5148,9 +5148,26 @@ class CostEstimateRequest(LiteLLMPydanticObjectBase):
model: str = Field(description="Model name (from /model_group/info)")
input_tokens: int = Field(description="Expected input tokens per request", ge=0)
output_tokens: int = Field(description="Expected output tokens per request", ge=0)
cache_read_input_tokens: int = Field(
default=0, description="Input tokens read from the prompt cache; counted within input_tokens", ge=0
)
cache_creation_input_tokens: int = Field(
default=0, description="Input tokens written to the prompt cache; counted within input_tokens", ge=0
)
reasoning_tokens: int = Field(
default=0, description="Reasoning tokens the model emits; counted within output_tokens", ge=0
)
num_requests_per_day: int | None = Field(default=None, description="Number of requests per day", ge=0)
num_requests_per_month: int | None = Field(default=None, description="Number of requests per month", ge=0)
@model_validator(mode="after")
def validate_token_subsets(self) -> "CostEstimateRequest":
if self.cache_read_input_tokens + self.cache_creation_input_tokens > self.input_tokens:
raise ValueError("cache_read_input_tokens plus cache_creation_input_tokens cannot exceed input_tokens")
if self.reasoning_tokens > self.output_tokens:
raise ValueError("reasoning_tokens cannot exceed output_tokens")
return self
class CostEstimateResponse(LiteLLMPydanticObjectBase):
"""Response body for /cost/estimate endpoint."""
@ -5158,6 +5175,9 @@ class CostEstimateResponse(LiteLLMPydanticObjectBase):
model: str
input_tokens: int
output_tokens: int
cache_read_input_tokens: int = 0
cache_creation_input_tokens: int = 0
reasoning_tokens: int = 0
num_requests_per_day: int | None = None
num_requests_per_month: int | None = None
# Per-request costs
@ -5165,17 +5185,33 @@ class CostEstimateResponse(LiteLLMPydanticObjectBase):
input_cost_per_request: float = Field(description="Input token cost per request (before margin)")
output_cost_per_request: float = Field(description="Output token cost per request (before margin)")
margin_cost_per_request: float = Field(default=0.0, description="Margin/fee added per request")
cache_read_cost_per_request: float = Field(default=0.0, description="Cache-read share of input_cost_per_request")
cache_creation_cost_per_request: float = Field(
default=0.0, description="Cache-write share of input_cost_per_request"
)
reasoning_cost_per_request: float = Field(default=0.0, description="Reasoning share of output_cost_per_request")
# Daily costs (if num_requests_per_day provided)
daily_cost: float | None = Field(default=None, description="Total daily cost (includes margin)")
daily_input_cost: float | None = Field(default=None, description="Daily input token cost")
daily_output_cost: float | None = Field(default=None, description="Daily output token cost")
daily_margin_cost: float | None = Field(default=None, description="Daily margin/fee")
daily_cache_read_cost: float | None = Field(default=None, description="Cache-read share of daily_input_cost")
daily_cache_creation_cost: float | None = Field(default=None, description="Cache-write share of daily_input_cost")
daily_reasoning_cost: float | None = Field(default=None, description="Reasoning share of daily_output_cost")
# Monthly costs (if num_requests_per_month provided)
monthly_cost: float | None = Field(default=None, description="Total monthly cost (includes margin)")
monthly_input_cost: float | None = Field(default=None, description="Monthly input token cost")
monthly_output_cost: float | None = Field(default=None, description="Monthly output token cost")
monthly_margin_cost: float | None = Field(default=None, description="Monthly margin/fee")
# Pricing info
input_cost_per_token: float | None = None
output_cost_per_token: float | None = None
monthly_cache_read_cost: float | None = Field(default=None, description="Cache-read share of monthly_input_cost")
monthly_cache_creation_cost: float | None = Field(
default=None, description="Cache-write share of monthly_input_cost"
)
monthly_reasoning_cost: float | None = Field(default=None, description="Reasoning share of monthly_output_cost")
# Pricing info: the rates this request's usage bills at, after token tiers and regional multipliers
input_cost_per_token: float | None = Field(default=None, description="Rate billed per input token")
output_cost_per_token: float | None = Field(default=None, description="Rate billed per output token")
cache_read_input_token_cost: float | None = Field(default=None, description="Rate billed per cache-read token")
cache_creation_input_token_cost: float | None = Field(default=None, description="Rate billed per cache-write token")
output_cost_per_reasoning_token: float | None = Field(default=None, description="Rate billed per reasoning token")
provider: str | None = None

258
litellm/proxy/management_endpoints/cost_tracking_settings.py
View file

@ -18,6 +18,7 @@ from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
import litellm
from litellm._internal_context import current_billing_time, pinned_billing_time
from litellm._logging import verbose_proxy_logger
from litellm.cost_calculator import completion_cost
from litellm.proxy._types import (
@ -27,7 +28,15 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.utils import CostPerToken, LlmProvidersSet, ModelInfo
from litellm.types.utils import (
CostBreakdown,
CostPerToken,
LlmProvidersSet,
ModelInfo,
ModelResponse,
PromptTokensDetailsWrapper,
Usage,
)
router: Final = APIRouter()
@ -46,13 +55,15 @@ def _configured_price(key: str, sources: tuple[Mapping[str, object], ...]) -> fl
def _extract_custom_pricing(
litellm_params: Mapping[str, object], model_info: Mapping[str, object]
litellm_params: Mapping[str, object], model_info: Mapping[str, object], builtin: ModelInfo | None
) -> CostPerToken | None:
"""
Pull per-token pricing configured on a deployment so on-prem / self-hosted
models (absent from the public cost map) still estimate a real cost.
Pricing may live on ``litellm_params`` or ``model_info``; ``litellm_params``
wins, matching the router's cost-map registration precedence.
wins, matching the router's cost-map registration precedence. Cache rates the
deployment leaves unset come from the backend model's built-in entry, then its
own input rate, again matching what the router registers for live billing.
"""
sources: Final = (litellm_params, model_info)
input_price: Final = _configured_price("input_cost_per_token", sources)
@ -61,15 +72,21 @@ def _extract_custom_pricing(
if input_price is None and output_price is None:
return None
input_rate: Final = input_price or 0.0
cache_sources: Final = sources if builtin is None else (*sources, builtin)
cache_read_price: Final = _configured_price("cache_read_input_token_cost", cache_sources)
cache_creation_price: Final = _configured_price("cache_creation_input_token_cost", cache_sources)
return CostPerToken(
input_cost_per_token=input_price or 0.0,
input_cost_per_token=input_rate,
output_cost_per_token=output_price or 0.0,
cache_read_input_token_cost=input_rate if cache_read_price is None else cache_read_price,
cache_creation_input_token_cost=input_rate if cache_creation_price is None else cache_creation_price,
)
def _lookup_model_info(model: str) -> ModelInfo | None:
def _lookup_model_info(model: str, custom_llm_provider: str | None = None) -> ModelInfo | None:
try:
return litellm.get_model_info(model=model)
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception:
return None
@ -98,17 +115,14 @@ def _resolve_model_for_cost_lookup(model: str) -> ResolvedCostModel:
model_info: Final = first_deployment.get("model_info", {})
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
provider: Final = str(custom_llm_provider) if custom_llm_provider is not None else None
custom_cost_per_token: Final = _extract_custom_pricing(litellm_params, model_info)
# Check base_model first (needed for Azure custom deployment names)
# base_model wins (needed for Azure custom deployment names)
base_model: Final = model_info.get("base_model") or litellm_params.get("base_model")
if base_model:
verbose_proxy_logger.debug("Resolved model '%s' to base_model '%s' from router", model, base_model)
return ResolvedCostModel(str(base_model), provider, custom_cost_per_token)
resolved_model: Final = litellm_params.get("model")
resolved_model: Final = base_model or litellm_params.get("model")
if resolved_model:
verbose_proxy_logger.debug("Resolved model '%s' to '%s' from router", model, resolved_model)
custom_cost_per_token: Final = _extract_custom_pricing(
litellm_params, model_info, _lookup_model_info(str(resolved_model), provider)
)
return ResolvedCostModel(str(resolved_model), provider, custom_cost_per_token)
except Exception as e:
verbose_proxy_logger.debug("Could not resolve model '%s' from router: %s", model, e)
@ -117,19 +131,59 @@ def _resolve_model_for_cost_lookup(model: str) -> ResolvedCostModel:
return ResolvedCostModel(model, None, None)
def _calculate_period_costs(num_requests, cost_per_request, input_cost, output_cost, margin_cost):
"""
Calculate costs for a given number of requests.
@dataclass(frozen=True, slots=True)
class CostLines:
"""Cost of one request split the way the spend logs split it: the cache lines are
shares of input_cost and the reasoning line is a share of output_cost."""
Returns tuple of (total_cost, input_cost, output_cost, margin_cost) or all None if num_requests is None/0.
"""
if not num_requests:
return None, None, None, None
return (
cost_per_request * num_requests,
input_cost * num_requests,
output_cost * num_requests,
margin_cost * num_requests,
total_cost: float
input_cost: float
output_cost: float
margin_cost: float
cache_read_cost: float
cache_creation_cost: float
reasoning_cost: float
def times(self, num_requests: int | None) -> "CostLines | None":
if not num_requests:
return None
return CostLines(
total_cost=self.total_cost * num_requests,
input_cost=self.input_cost * num_requests,
output_cost=self.output_cost * num_requests,
margin_cost=self.margin_cost * num_requests,
cache_read_cost=self.cache_read_cost * num_requests,
cache_creation_cost=self.cache_creation_cost * num_requests,
reasoning_cost=self.reasoning_cost * num_requests,
)
def _cost_lines(cost_per_request: float, cost_breakdown: CostBreakdown | None) -> CostLines:
breakdown: Final = cost_breakdown if cost_breakdown is not None else CostBreakdown()
return CostLines(
total_cost=cost_per_request,
input_cost=breakdown.get("input_cost", 0.0),
output_cost=breakdown.get("output_cost", 0.0),
margin_cost=breakdown.get("margin_total_amount", 0.0),
cache_read_cost=breakdown.get("cache_read_cost", 0.0),
cache_creation_cost=breakdown.get("cache_creation_cost", 0.0),
reasoning_cost=breakdown.get("reasoning_cost", 0.0),
)
def _usage_for_estimate(request: CostEstimateRequest) -> Usage:
cache_tokens: Final = request.cache_read_input_tokens + request.cache_creation_input_tokens
return Usage(
prompt_tokens=request.input_tokens,
completion_tokens=request.output_tokens,
total_tokens=request.input_tokens + request.output_tokens,
reasoning_tokens=request.reasoning_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=request.cache_read_input_tokens,
cache_creation_tokens=request.cache_creation_input_tokens,
)
if cache_tokens
else None,
)
@ -530,11 +584,14 @@ async def estimate_cost(
- model: Model name (e.g., "gpt-4", "claude-3-opus")
- input_tokens: Expected input tokens per request
- output_tokens: Expected output tokens per request
- cache_read_input_tokens: Cache-read tokens per request, counted within input_tokens (optional)
- cache_creation_input_tokens: Cache-write tokens per request, counted within input_tokens (optional)
- reasoning_tokens: Reasoning tokens per request, counted within output_tokens (optional)
- num_requests_per_day: Number of requests per day (optional)
- num_requests_per_month: Number of requests per month (optional)
Returns cost breakdown including:
- Per-request costs (input, output, margin)
- Per-request costs (input, output, margin, plus the cache-read, cache-write and reasoning shares)
- Daily costs (if num_requests_per_day provided)
- Monthly costs (if num_requests_per_month provided)
@ -543,14 +600,15 @@ async def estimate_cost(
{
"model": "gpt-4",
"input_tokens": 1000,
"cache_read_input_tokens": 800,
"output_tokens": 500,
"reasoning_tokens": 200,
"num_requests_per_day": 100,
"num_requests_per_month": 3000
}
```
"""
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.utils import ModelResponse, Usage
# Resolve model name (handles router aliases like 'e-model-router' -> 'azure_ai/gpt-4')
resolved: Final = _resolve_model_for_cost_lookup(request.model)
@ -559,15 +617,8 @@ async def estimate_cost(
verbose_proxy_logger.debug("Cost estimate: request.model='%s' resolved to '%s'", request.model, resolved_model)
# Create a mock response with usage for completion_cost
mock_response: Final = ModelResponse(
model=resolved_model,
usage=Usage(
prompt_tokens=request.input_tokens,
completion_tokens=request.output_tokens,
total_tokens=request.input_tokens + request.output_tokens,
),
)
usage: Final = _usage_for_estimate(request)
mock_response: Final = ModelResponse(model=resolved_model, usage=usage)
# Create a logging object to capture cost breakdown
litellm_logging_obj: Final = LiteLLMLoggingObj(
@ -580,92 +631,73 @@ async def estimate_cost(
function_id="cost-estimate",
)
# Use completion_cost which handles all the logic including margins/discounts
try:
cost_per_request: Final = completion_cost(
completion_response=mock_response,
model=resolved_model,
custom_llm_provider=resolved_provider,
custom_cost_per_token=resolved.custom_cost_per_token,
litellm_logging_obj=litellm_logging_obj,
)
except Exception as e:
raise HTTPException(
status_code=404,
detail={
"error": f"Could not calculate cost for model '{request.model}' (resolved to '{resolved_model}'): {e}"
},
)
# Pinning one moment keeps an off-peak window that opens mid-quote from pricing the totals on
# one side of it and the reported rates on the other.
with pinned_billing_time(current_billing_time()):
# Use completion_cost which handles all the logic including margins/discounts
try:
cost_per_request: Final = completion_cost(
completion_response=mock_response,
model=resolved_model,
custom_llm_provider=resolved_provider,
custom_cost_per_token=resolved.custom_cost_per_token,
litellm_logging_obj=litellm_logging_obj,
)
except Exception as e:
raise HTTPException(
status_code=404,
detail={
"error": f"Could not calculate cost for model '{request.model}' (resolved to '{resolved_model}'): {e}"
},
)
# Get cost breakdown from the logging object
cost_breakdown: Final = litellm_logging_obj.cost_breakdown
# The rates come back from the pricing call itself rather than a second lookup, so they are the
# ones the cost lines above billed at even when completion_cost infers a provider this endpoint
# never resolved (an unrouted "xai/grok-4" prices on xai's inclusive tier thresholds; a lookup
# here without that provider would report the sub-200k rate for a line billed above it).
rates: Final = litellm_logging_obj.billed_token_rates
per_request: Final = _cost_lines(cost_per_request, litellm_logging_obj.cost_breakdown)
daily: Final = per_request.times(request.num_requests_per_day)
monthly: Final = per_request.times(request.num_requests_per_month)
input_cost: Final = cost_breakdown.get("input_cost", 0.0) if cost_breakdown else 0.0
output_cost: Final = cost_breakdown.get("output_cost", 0.0) if cost_breakdown else 0.0
margin_cost: Final = cost_breakdown.get("margin_total_amount", 0.0) if cost_breakdown else 0.0
model_info: Final = _lookup_model_info(resolved_model)
mapped_input_price: Final = model_info.get("input_cost_per_token") if model_info is not None else None
mapped_output_price: Final = model_info.get("output_cost_per_token") if model_info is not None else None
model_info: Final = _lookup_model_info(resolved_model, resolved_provider)
mapped_provider: Final = model_info.get("litellm_provider") if model_info is not None else None
input_cost_per_token: Final = (
resolved.custom_cost_per_token["input_cost_per_token"]
if resolved.custom_cost_per_token is not None
else mapped_input_price
)
output_cost_per_token: Final = (
resolved.custom_cost_per_token["output_cost_per_token"]
if resolved.custom_cost_per_token is not None
else mapped_output_price
)
custom_llm_provider: Final = mapped_provider if mapped_provider is not None else resolved_provider
# Calculate daily and monthly costs
(
daily_cost,
daily_input_cost,
daily_output_cost,
daily_margin_cost,
) = _calculate_period_costs(
num_requests=request.num_requests_per_day,
cost_per_request=cost_per_request,
input_cost=input_cost,
output_cost=output_cost,
margin_cost=margin_cost,
)
(
monthly_cost,
monthly_input_cost,
monthly_output_cost,
monthly_margin_cost,
) = _calculate_period_costs(
num_requests=request.num_requests_per_month,
cost_per_request=cost_per_request,
input_cost=input_cost,
output_cost=output_cost,
margin_cost=margin_cost,
)
return CostEstimateResponse(
model=request.model,
input_tokens=request.input_tokens,
output_tokens=request.output_tokens,
cache_read_input_tokens=request.cache_read_input_tokens,
cache_creation_input_tokens=request.cache_creation_input_tokens,
reasoning_tokens=request.reasoning_tokens,
num_requests_per_day=request.num_requests_per_day,
num_requests_per_month=request.num_requests_per_month,
cost_per_request=cost_per_request,
input_cost_per_request=input_cost,
output_cost_per_request=output_cost,
margin_cost_per_request=margin_cost,
daily_cost=daily_cost,
daily_input_cost=daily_input_cost,
daily_output_cost=daily_output_cost,
daily_margin_cost=daily_margin_cost,
monthly_cost=monthly_cost,
monthly_input_cost=monthly_input_cost,
monthly_output_cost=monthly_output_cost,
monthly_margin_cost=monthly_margin_cost,
input_cost_per_token=input_cost_per_token,
output_cost_per_token=output_cost_per_token,
cost_per_request=per_request.total_cost,
input_cost_per_request=per_request.input_cost,
output_cost_per_request=per_request.output_cost,
margin_cost_per_request=per_request.margin_cost,
cache_read_cost_per_request=per_request.cache_read_cost,
cache_creation_cost_per_request=per_request.cache_creation_cost,
reasoning_cost_per_request=per_request.reasoning_cost,
daily_cost=daily.total_cost if daily is not None else None,
daily_input_cost=daily.input_cost if daily is not None else None,
daily_output_cost=daily.output_cost if daily is not None else None,
daily_margin_cost=daily.margin_cost if daily is not None else None,
daily_cache_read_cost=daily.cache_read_cost if daily is not None else None,
daily_cache_creation_cost=daily.cache_creation_cost if daily is not None else None,
daily_reasoning_cost=daily.reasoning_cost if daily is not None else None,
monthly_cost=monthly.total_cost if monthly is not None else None,
monthly_input_cost=monthly.input_cost if monthly is not None else None,
monthly_output_cost=monthly.output_cost if monthly is not None else None,
monthly_margin_cost=monthly.margin_cost if monthly is not None else None,
monthly_cache_read_cost=monthly.cache_read_cost if monthly is not None else None,
monthly_cache_creation_cost=monthly.cache_creation_cost if monthly is not None else None,
monthly_reasoning_cost=monthly.reasoning_cost if monthly is not None else None,
input_cost_per_token=rates.input_cost_per_token if rates is not None else None,
output_cost_per_token=rates.output_cost_per_token if rates is not None else None,
cache_read_input_token_cost=rates.cache_read_input_token_cost if rates is not None else None,
cache_creation_input_token_cost=rates.cache_creation_input_token_cost if rates is not None else None,
output_cost_per_reasoning_token=rates.output_cost_per_reasoning_token if rates is not None else None,
provider=custom_llm_provider,
)

207
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -1,9 +1,11 @@
import json
from datetime import datetime, timezone
import pytest
from fastapi.testclient import TestClient
import litellm
from litellm._internal_context import pinned_billing_time
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
@ -27,10 +29,10 @@ from litellm.types.utils import (
)
from litellm.litellm_core_utils.llm_cost_calc.utils import (
BilledTokenRates,
CostCalculatorUtils,
PromptTokensDetailsResult,
TokenRates,
TokenTypeCostBreakdown,
_calculate_input_cost,
_get_token_base_cost,
_is_off_peak,
@ -38,6 +40,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
apply_off_peak_pricing,
calculate_cache_writing_cost,
generic_cost_per_token,
get_billed_token_rates,
get_token_type_cost_breakdown,
)
from litellm.types.utils import CacheCreationTokenDetails, Usage
@ -3906,6 +3909,200 @@ def test_token_type_cost_breakdown_reconciles_with_generic_total(_local_model_co
assert text_input_cost + breakdown.cache_read_cost == pytest.approx(prompt_cost)
def _custom_priced_usage() -> Usage:
return Usage(
prompt_tokens=1000,
completion_tokens=500,
total_tokens=1500,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800, cache_creation_tokens=100),
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200),
)
def test_token_type_cost_breakdown_prices_custom_pricing_from_its_flat_rates():
"""
A custom-priced deployment, usually absent from the cost map, used to get zero cache and
reasoning lines while its total already billed cache tokens at the custom cache rates.
The lines must come from the same flat rates: a configured cache rate, else the input
rate for cache tokens and the output rate for reasoning tokens.
"""
from litellm.types.utils import CostPerToken
breakdown = get_token_type_cost_breakdown(
model="openai/onprem-model",
custom_llm_provider="openai",
usage=_custom_priced_usage(),
custom_cost_per_token=CostPerToken(
input_cost_per_token=1e-6, output_cost_per_token=2e-6, cache_read_input_token_cost=1e-7
),
)
assert breakdown.cache_read_cost == pytest.approx(800 * 1e-7)
assert breakdown.cache_creation_cost == pytest.approx(100 * 1e-6)
assert breakdown.reasoning_cost == pytest.approx(200 * 2e-6)
def test_token_type_cost_breakdown_reconciles_with_custom_pricing_totals():
from litellm.cost_calculator import cost_per_token
from litellm.types.utils import CostPerToken
usage = _custom_priced_usage()
custom_cost_per_token = CostPerToken(
input_cost_per_token=1e-6,
output_cost_per_token=2e-6,
cache_read_input_token_cost=1e-7,
cache_creation_input_token_cost=1.25e-6,
)
prompt_cost, completion_cost = cost_per_token(
model="openai/onprem-model",
custom_llm_provider="openai",
prompt_tokens=1000,
completion_tokens=500,
usage_object=usage,
custom_cost_per_token=custom_cost_per_token,
)
breakdown = get_token_type_cost_breakdown(
model="openai/onprem-model",
custom_llm_provider="openai",
usage=usage,
custom_cost_per_token=custom_cost_per_token,
)
assert 100 * 1e-6 + breakdown.cache_read_cost + breakdown.cache_creation_cost == pytest.approx(prompt_cost)
assert 300 * 2e-6 + breakdown.reasoning_cost == pytest.approx(completion_cost)
def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
"tiered-cache-model",
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"cache_read_input_token_cost": 3e-7,
"cache_creation_input_token_cost": 3.75e-6,
"input_cost_per_token_above_200k_tokens": 6e-6,
"output_cost_per_token_above_200k_tokens": 3e-5,
"cache_read_input_token_cost_above_200k_tokens": 6e-7,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-6,
"litellm_provider": "openai",
"mode": "chat",
},
)
usage = Usage(
prompt_tokens=250_000,
completion_tokens=1_000,
total_tokens=251_000,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200_000, cache_creation_tokens=10_000),
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200),
)
rates = get_billed_token_rates(model="tiered-cache-model", custom_llm_provider="openai", usage=usage)
breakdown = get_token_type_cost_breakdown(model="tiered-cache-model", custom_llm_provider="openai", usage=usage)
assert rates == BilledTokenRates(
input_cost_per_token=6e-6,
output_cost_per_token=3e-5,
cache_read_input_token_cost=6e-7,
cache_creation_input_token_cost=7.5e-6,
cache_creation_input_token_cost_above_1hr=0.0,
output_cost_per_reasoning_token=3e-5,
)
assert breakdown.cache_read_cost == pytest.approx(200_000 * rates.cache_read_input_token_cost)
assert breakdown.cache_creation_cost == pytest.approx(10_000 * rates.cache_creation_input_token_cost)
assert breakdown.reasoning_cost == pytest.approx(200 * rates.output_cost_per_reasoning_token)
def test_a_pinned_billing_time_prices_the_totals_and_the_reported_rates_at_one_moment(monkeypatch):
"""Totals and reported rates resolve off-peak pricing on separate paths that each read the
clock, so a window opening between the two reads used to leave them describing one request
at two different prices. Pinned, both must answer for the pinned moment."""
monkeypatch.setitem(
litellm.model_cost,
"off-peak-model",
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"off_peak_pricing": {
"hours_utc": "02:00-03:00",
"input_cost_per_token": 1e-6,
"output_cost_per_token": 5e-6,
},
"litellm_provider": "openai",
"mode": "chat",
},
)
usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)):
off_peak_prompt_cost, off_peak_completion_cost = generic_cost_per_token(
model="off-peak-model", usage=usage, custom_llm_provider="openai"
)
off_peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage)
with pinned_billing_time(datetime(2026, 1, 1, 12, 30, tzinfo=timezone.utc)):
peak_prompt_cost, peak_completion_cost = generic_cost_per_token(
model="off-peak-model", usage=usage, custom_llm_provider="openai"
)
peak_rates = get_billed_token_rates(model="off-peak-model", custom_llm_provider="openai", usage=usage)
assert off_peak_rates.input_cost_per_token == pytest.approx(1e-6)
assert peak_rates.input_cost_per_token == pytest.approx(3e-6)
assert off_peak_prompt_cost == pytest.approx(1000 * off_peak_rates.input_cost_per_token)
assert off_peak_completion_cost == pytest.approx(500 * off_peak_rates.output_cost_per_token)
assert peak_prompt_cost == pytest.approx(1000 * peak_rates.input_cost_per_token)
assert peak_completion_cost == pytest.approx(500 * peak_rates.output_cost_per_token)
def test_the_token_type_breakdown_carries_the_rates_it_billed_at(monkeypatch):
"""Callers that report both the lines and the rates read the rates off the breakdown rather than
resolving them a second time, so the breakdown has to hand back exactly what it billed at."""
monkeypatch.setitem(
litellm.model_cost,
"xai/tiered-model",
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"cache_read_input_token_cost": 3e-7,
"input_cost_per_token_above_200k_tokens": 6e-6,
"output_cost_per_token_above_200k_tokens": 3e-5,
"cache_read_input_token_cost_above_200k_tokens": 6e-7,
"litellm_provider": "xai",
"mode": "chat",
},
)
usage = Usage(
prompt_tokens=200_000,
completion_tokens=1_000,
total_tokens=201_000,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100_000),
)
breakdown = get_token_type_cost_breakdown(model="xai/tiered-model", custom_llm_provider="xai", usage=usage)
assert breakdown.rates == get_billed_token_rates(
model="xai/tiered-model", custom_llm_provider="xai", usage=usage
)
assert breakdown.rates.cache_read_input_token_cost == pytest.approx(6e-7)
assert breakdown.cache_read_cost == pytest.approx(100_000 * breakdown.rates.cache_read_input_token_cost)
def test_the_token_type_breakdown_reports_no_rates_for_an_unpriced_model():
usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
breakdown = get_token_type_cost_breakdown(
model="no-such-model-anywhere", custom_llm_provider="openai", usage=usage
)
assert breakdown.rates is None
def test_billed_token_rates_are_none_for_an_unpriced_model():
usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
assert get_billed_token_rates(model="no-such-model-anywhere", custom_llm_provider="openai", usage=usage) is None
def test_token_type_cost_breakdown_zero_without_special_tokens(_local_model_cost_map):
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
@ -3913,9 +4110,7 @@ def test_token_type_cost_breakdown_zero_without_special_tokens(_local_model_cost
model="gpt-4o", custom_llm_provider="openai", usage=usage
)
assert breakdown == TokenTypeCostBreakdown(
reasoning_cost=0.0, cache_read_cost=0.0, cache_creation_cost=0.0
)
assert (breakdown.reasoning_cost, breakdown.cache_read_cost, breakdown.cache_creation_cost) == (0.0, 0.0, 0.0)
@pytest.mark.parametrize(
@ -3987,9 +4182,7 @@ def test_token_type_cost_breakdown_handles_unknown_model_gracefully():
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=5),
),
)
assert breakdown == TokenTypeCostBreakdown(
reasoning_cost=0.0, cache_read_cost=0.0, cache_creation_cost=0.0
)
assert (breakdown.reasoning_cost, breakdown.cache_read_cost, breakdown.cache_creation_cost) == (0.0, 0.0, 0.0)
def test_token_type_cost_breakdown_applies_regional_uplift(_local_model_cost_map):

308
tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py
View file

@ -4,13 +4,17 @@ Tests for cost tracking settings management endpoints.
Tests the GET and PATCH endpoints for managing cost discount configuration.
"""
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi.testclient import TestClient
from pydantic import ValidationError
import litellm
from litellm._internal_context import pinned_billing_time
from litellm.proxy._types import CostEstimateRequest
from litellm.proxy.management_endpoints.cost_tracking_settings import router
from litellm.proxy.proxy_server import app
@ -789,13 +793,13 @@ INPUT_TOKENS = 1000
OUTPUT_TOKENS = 500
def _router_pricing(**pricing: float) -> MagicMock:
def _router_pricing(model: str = AN_UNDERLYING_MODEL, **pricing: float) -> MagicMock:
mock_router = MagicMock()
mock_router.get_model_list.return_value = [
{
"model_name": AN_ALIAS,
"litellm_params": {
"model": AN_UNDERLYING_MODEL,
"model": model,
"custom_llm_provider": "openai",
**pricing,
},
@ -811,9 +815,7 @@ async def _estimate(mock_router: MagicMock | None, model: str = AN_ALIAS, **over
request = CostEstimateRequest(
model=model,
input_tokens=INPUT_TOKENS,
output_tokens=OUTPUT_TOKENS,
**overrides,
**{"input_tokens": INPUT_TOKENS, "output_tokens": OUTPUT_TOKENS, **overrides},
)
with patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point
"litellm.proxy.proxy_server.llm_router", mock_router
@ -909,3 +911,299 @@ class TestEstimateCostPeriodTotals:
assert response.cost_per_request == pytest.approx(0.0022)
assert response.daily_margin_cost == pytest.approx(0.02)
assert response.daily_cost == pytest.approx(0.22)
CACHE_READ_TOKENS = 800
CACHE_CREATION_TOKENS = 100
REASONING_TOKENS = 200
TEXT_INPUT_TOKENS = INPUT_TOKENS - CACHE_READ_TOKENS - CACHE_CREATION_TOKENS
TEXT_OUTPUT_TOKENS = OUTPUT_TOKENS - REASONING_TOKENS
async def _estimate_with_cache_and_reasoning(mock_router: MagicMock | None, model: str = AN_ALIAS, **overrides: int):
return await _estimate(
mock_router,
model=model,
cache_read_input_tokens=CACHE_READ_TOKENS,
cache_creation_input_tokens=CACHE_CREATION_TOKENS,
reasoning_tokens=REASONING_TOKENS,
**overrides,
)
class TestEstimateCostCacheAndReasoningTokens:
@pytest.mark.asyncio
async def test_a_mapped_model_bills_cache_and_reasoning_tokens_at_their_own_rates(self, monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"cache_read_input_token_cost": 3e-7,
"cache_creation_input_token_cost": 3.75e-6,
"output_cost_per_reasoning_token": 1e-5,
"litellm_provider": "openai",
"mode": "chat",
},
)
response = await _estimate_with_cache_and_reasoning(None, model=A_MAPPED_MODEL, num_requests_per_day=10)
assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 3e-7)
assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 3.75e-6)
assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 1e-5)
assert response.input_cost_per_request == pytest.approx(
TEXT_INPUT_TOKENS * 3e-6 + CACHE_READ_TOKENS * 3e-7 + CACHE_CREATION_TOKENS * 3.75e-6
)
assert response.output_cost_per_request == pytest.approx(TEXT_OUTPUT_TOKENS * 15e-6 + REASONING_TOKENS * 1e-5)
assert response.cost_per_request == pytest.approx(
response.input_cost_per_request + response.output_cost_per_request
)
assert response.daily_cache_read_cost == pytest.approx(10 * CACHE_READ_TOKENS * 3e-7)
assert response.daily_cache_creation_cost == pytest.approx(10 * CACHE_CREATION_TOKENS * 3.75e-6)
assert response.daily_reasoning_cost == pytest.approx(10 * REASONING_TOKENS * 1e-5)
assert response.monthly_cache_read_cost is None
assert response.cache_read_input_token_cost == pytest.approx(3e-7)
assert response.cache_creation_input_token_cost == pytest.approx(3.75e-6)
assert response.output_cost_per_reasoning_token == pytest.approx(1e-5)
assert (
response.cache_read_input_tokens,
response.cache_creation_input_tokens,
response.reasoning_tokens,
) == (CACHE_READ_TOKENS, CACHE_CREATION_TOKENS, REASONING_TOKENS)
@pytest.mark.asyncio
async def test_a_model_without_cache_or_reasoning_prices_estimates_what_the_proxy_bills(self, monkeypatch):
"""The cost calculator bills cache tokens of a cost-map model without cache prices at zero
and its reasoning tokens at the output rate. The estimate reports those effective rates."""
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{"input_cost_per_token": 5e-6, "output_cost_per_token": 6e-6, "litellm_provider": "openai", "mode": "chat"},
)
response = await _estimate_with_cache_and_reasoning(None, model=A_MAPPED_MODEL)
assert response.cache_read_cost_per_request == 0.0
assert response.cache_creation_cost_per_request == 0.0
assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 6e-6)
assert response.input_cost_per_request == pytest.approx(TEXT_INPUT_TOKENS * 5e-6)
assert response.cost_per_request == pytest.approx(TEXT_INPUT_TOKENS * 5e-6 + OUTPUT_TOKENS * 6e-6)
assert response.cache_read_input_token_cost == 0.0
assert response.cache_creation_input_token_cost == 0.0
assert response.output_cost_per_reasoning_token == pytest.approx(6e-6)
@pytest.mark.asyncio
async def test_a_request_without_cache_or_reasoning_tokens_estimates_as_before(self, monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"cache_read_input_token_cost": 3e-7,
"cache_creation_input_token_cost": 3.75e-6,
"output_cost_per_reasoning_token": 1e-5,
"litellm_provider": "openai",
"mode": "chat",
},
)
response = await _estimate(None, model=A_MAPPED_MODEL, num_requests_per_day=10)
assert response.cost_per_request == pytest.approx(INPUT_TOKENS * 3e-6 + OUTPUT_TOKENS * 15e-6)
assert response.cache_read_cost_per_request == 0.0
assert response.cache_creation_cost_per_request == 0.0
assert response.reasoning_cost_per_request == 0.0
assert response.daily_cache_read_cost == 0.0
assert response.daily_reasoning_cost == 0.0
@pytest.mark.asyncio
async def test_a_custom_priced_deployment_bills_cache_and_reasoning_tokens_from_its_flat_rates(self):
response = await _estimate_with_cache_and_reasoning(
_router_pricing(input_cost_per_token=1e-6, output_cost_per_token=2e-6, cache_read_input_token_cost=1e-7)
)
assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 1e-7)
assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 1e-6)
assert response.reasoning_cost_per_request == pytest.approx(REASONING_TOKENS * 2e-6)
assert response.cost_per_request == pytest.approx(
TEXT_INPUT_TOKENS * 1e-6 + CACHE_READ_TOKENS * 1e-7 + CACHE_CREATION_TOKENS * 1e-6 + OUTPUT_TOKENS * 2e-6
)
assert response.cache_read_input_token_cost == pytest.approx(1e-7)
assert response.cache_creation_input_token_cost == pytest.approx(1e-6)
assert response.output_cost_per_reasoning_token == pytest.approx(2e-6)
@pytest.mark.asyncio
async def test_a_custom_priced_deployment_of_a_mapped_model_inherits_its_built_in_cache_rates(self, monkeypatch):
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{
"input_cost_per_token": 5e-6,
"output_cost_per_token": 6e-6,
"cache_read_input_token_cost": 5e-7,
"cache_creation_input_token_cost": 6.25e-6,
"litellm_provider": "openai",
"mode": "chat",
},
)
response = await _estimate_with_cache_and_reasoning(
_router_pricing(model=A_MAPPED_MODEL, input_cost_per_token=1e-6, output_cost_per_token=2e-6)
)
assert response.cache_read_cost_per_request == pytest.approx(CACHE_READ_TOKENS * 5e-7)
assert response.cache_creation_cost_per_request == pytest.approx(CACHE_CREATION_TOKENS * 6.25e-6)
assert response.input_cost_per_request == pytest.approx(
TEXT_INPUT_TOKENS * 1e-6 + CACHE_READ_TOKENS * 5e-7 + CACHE_CREATION_TOKENS * 6.25e-6
)
assert response.cache_read_input_token_cost == pytest.approx(5e-7)
assert response.cache_creation_input_token_cost == pytest.approx(6.25e-6)
@pytest.mark.asyncio
async def test_a_tiered_model_reports_the_rates_its_lines_were_billed_at(self, monkeypatch):
"""Above a token tier the calculator bills every line at the tier's rate, so the reported
rates must be the tier's too: each line equals its token count times the rate next to it."""
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"cache_read_input_token_cost": 3e-7,
"cache_creation_input_token_cost": 3.75e-6,
"input_cost_per_token_above_200k_tokens": 6e-6,
"output_cost_per_token_above_200k_tokens": 3e-5,
"cache_read_input_token_cost_above_200k_tokens": 6e-7,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-6,
"litellm_provider": "openai",
"mode": "chat",
},
)
response = await _estimate(
None,
model=A_MAPPED_MODEL,
input_tokens=250_000,
cache_read_input_tokens=200_000,
cache_creation_input_tokens=10_000,
output_tokens=1_000,
reasoning_tokens=200,
)
assert response.input_cost_per_token == pytest.approx(6e-6)
assert response.output_cost_per_token == pytest.approx(3e-5)
assert response.cache_read_input_token_cost == pytest.approx(6e-7)
assert response.cache_creation_input_token_cost == pytest.approx(7.5e-6)
assert response.output_cost_per_reasoning_token == pytest.approx(3e-5)
assert response.cache_read_cost_per_request == pytest.approx(200_000 * response.cache_read_input_token_cost)
assert response.cache_creation_cost_per_request == pytest.approx(
10_000 * response.cache_creation_input_token_cost
)
assert response.reasoning_cost_per_request == pytest.approx(200 * response.output_cost_per_reasoning_token)
assert response.input_cost_per_request == pytest.approx(
40_000 * response.input_cost_per_token
+ response.cache_read_cost_per_request
+ response.cache_creation_cost_per_request
)
assert response.output_cost_per_request == pytest.approx(1_000 * response.output_cost_per_token)
@pytest.mark.asyncio
async def test_a_quote_prices_its_totals_and_its_rates_at_the_same_moment(self, monkeypatch):
"""The totals and the reported rates resolve off-peak pricing on separate paths. A quote
taken as a window opens must not bill on one side of it and report rates from the other."""
monkeypatch.setitem(
litellm.model_cost,
A_MAPPED_MODEL,
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"off_peak_pricing": {
"hours_utc": "02:00-03:00",
"input_cost_per_token": 1e-6,
"output_cost_per_token": 5e-6,
},
"litellm_provider": "openai",
"mode": "chat",
},
)
with pinned_billing_time(datetime(2026, 1, 1, 2, 30, tzinfo=timezone.utc)):
response = await _estimate(None, model=A_MAPPED_MODEL)
assert response.input_cost_per_token == pytest.approx(1e-6)
assert response.output_cost_per_token == pytest.approx(5e-6)
assert response.input_cost_per_request == pytest.approx(INPUT_TOKENS * response.input_cost_per_token)
assert response.output_cost_per_request == pytest.approx(OUTPUT_TOKENS * response.output_cost_per_token)
@pytest.mark.asyncio
async def test_an_unrouted_model_reports_the_rates_of_the_provider_the_calculator_inferred(self, monkeypatch):
"""The cost calculator infers a provider this endpoint never resolved, and the provider decides
whether a tier threshold is inclusive. xai bills a request sitting exactly on the 200k threshold
at the tier rate, so the reported rates have to be the tier's rather than the sub-tier base."""
an_xai_model = "xai/tiered-model"
monkeypatch.setitem(
litellm.model_cost,
an_xai_model,
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"cache_read_input_token_cost": 3e-7,
"input_cost_per_token_above_200k_tokens": 6e-6,
"output_cost_per_token_above_200k_tokens": 3e-5,
"cache_read_input_token_cost_above_200k_tokens": 6e-7,
"litellm_provider": "xai",
"mode": "chat",
},
)
response = await _estimate(
None,
model=an_xai_model,
input_tokens=200_000,
cache_read_input_tokens=100_000,
output_tokens=1_000,
)
assert response.input_cost_per_token == pytest.approx(6e-6)
assert response.output_cost_per_token == pytest.approx(3e-5)
assert response.cache_read_input_token_cost == pytest.approx(6e-7)
assert response.cache_read_cost_per_request == pytest.approx(100_000 * response.cache_read_input_token_cost)
assert response.input_cost_per_request == pytest.approx(
100_000 * response.input_cost_per_token + response.cache_read_cost_per_request
)
assert response.output_cost_per_request == pytest.approx(1_000 * response.output_cost_per_token)
class TestCostEstimateRequestTokenSubsets:
def test_cache_tokens_beyond_the_input_tokens_are_rejected(self):
with pytest.raises(ValidationError, match="cannot exceed input_tokens"):
CostEstimateRequest(
model=AN_ALIAS,
input_tokens=INPUT_TOKENS,
output_tokens=OUTPUT_TOKENS,
cache_read_input_tokens=INPUT_TOKENS,
cache_creation_input_tokens=1,
)
def test_reasoning_tokens_beyond_the_output_tokens_are_rejected(self):
with pytest.raises(ValidationError, match="cannot exceed output_tokens"):
CostEstimateRequest(
model=AN_ALIAS,
input_tokens=INPUT_TOKENS,
output_tokens=OUTPUT_TOKENS,
reasoning_tokens=OUTPUT_TOKENS + 1,
)
def test_the_endpoint_answers_422_when_cache_tokens_exceed_input_tokens(self):
response = client.post(
"/cost/estimate",
headers={"Authorization": "Bearer sk-1234"},
json={"model": AN_ALIAS, "input_tokens": 1000, "output_tokens": 100, "cache_read_input_tokens": 8000},
)
assert response.status_code == 422
assert "cannot exceed input_tokens" in response.text

106
tests/test_litellm/test_cost_calculator.py
View file

@ -3736,6 +3736,112 @@ def test_completion_cost_logs_reasoning_and_cache_breakdown(_local_model_cost_ma
assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(100 * 3e-08)
def test_completion_cost_logs_the_rates_it_billed_at(monkeypatch):
"""A caller reporting the cost lines beside their per-token rates reads both off this one call.
completion_cost infers the provider, and xai's inclusive tier thresholds put a request sitting
exactly on 200k at the tier rate, which a lookup made without that inferred provider would miss.
"""
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import Logging
monkeypatch.setitem(
litellm.model_cost,
"xai/tiered-model",
{
"input_cost_per_token": 3e-6,
"output_cost_per_token": 15e-6,
"cache_read_input_token_cost": 3e-7,
"input_cost_per_token_above_200k_tokens": 6e-6,
"output_cost_per_token_above_200k_tokens": 3e-5,
"cache_read_input_token_cost_above_200k_tokens": 6e-7,
"litellm_provider": "xai",
"mode": "chat",
},
)
logging_obj = Logging(
model="xai/tiered-model",
messages=[{"role": "user", "content": "Hello"}],
stream=False,
call_type="completion",
start_time=datetime.now(),
litellm_call_id="billed-rates",
function_id="f",
)
usage = Usage(
prompt_tokens=200_000,
completion_tokens=1_000,
total_tokens=201_000,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=100_000),
)
litellm.completion_cost(
completion_response=ModelResponse(model="xai/tiered-model", usage=usage),
model="xai/tiered-model",
custom_llm_provider=None,
litellm_logging_obj=logging_obj,
)
rates = logging_obj.billed_token_rates
assert rates is not None
assert rates.input_cost_per_token == pytest.approx(6e-6)
assert rates.cache_read_input_token_cost == pytest.approx(6e-7)
assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(
100_000 * rates.cache_read_input_token_cost
)
assert logging_obj.cost_breakdown["output_cost"] == pytest.approx(1_000 * rates.output_cost_per_token)
def test_completion_cost_logs_cache_and_reasoning_breakdown_for_custom_pricing():
"""
A custom-priced deployment bills cache tokens at its custom cache rates, but the
breakdown stored for the spend logs carried no cache or reasoning lines for it.
"""
from datetime import datetime
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.utils import CompletionTokensDetailsWrapper, CostPerToken
logging_obj = Logging(
model="openai/onprem-model",
messages=[{"role": "user", "content": "Hello"}],
stream=False,
call_type="completion",
start_time=datetime.now(),
litellm_call_id="custom-pricing-breakdown",
function_id="f",
)
response = ModelResponse(
model="openai/onprem-model",
usage=Usage(
prompt_tokens=1000,
completion_tokens=500,
total_tokens=1500,
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800, cache_creation_tokens=100),
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200),
),
)
total = completion_cost(
completion_response=response,
model="openai/onprem-model",
custom_llm_provider="openai",
custom_cost_per_token=CostPerToken(
input_cost_per_token=1e-6,
output_cost_per_token=2e-6,
cache_read_input_token_cost=1e-7,
cache_creation_input_token_cost=1.25e-6,
),
litellm_logging_obj=logging_obj,
)
assert logging_obj.cost_breakdown is not None
assert logging_obj.cost_breakdown["cache_read_cost"] == pytest.approx(800 * 1e-7)
assert logging_obj.cost_breakdown["cache_creation_cost"] == pytest.approx(100 * 1.25e-6)
assert logging_obj.cost_breakdown["reasoning_cost"] == pytest.approx(200 * 2e-6)
assert total == pytest.approx(100 * 1e-6 + 800 * 1e-7 + 100 * 1.25e-6 + 500 * 2e-6)
def test_cost_per_token_per_second_pricing(monkeypatch):
"""
Models priced by duration (input/output_cost_per_second) with no per-token rates

113
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -3318,11 +3318,14 @@ export interface paths {
* - model: Model name (e.g., "gpt-4", "claude-3-opus")
* - input_tokens: Expected input tokens per request
* - output_tokens: Expected output tokens per request
* - cache_read_input_tokens: Cache-read tokens per request, counted within input_tokens (optional)
* - cache_creation_input_tokens: Cache-write tokens per request, counted within input_tokens (optional)
* - reasoning_tokens: Reasoning tokens per request, counted within output_tokens (optional)
* - num_requests_per_day: Number of requests per day (optional)
* - num_requests_per_month: Number of requests per month (optional)
*
* Returns cost breakdown including:
* - Per-request costs (input, output, margin)
* - Per-request costs (input, output, margin, plus the cache-read, cache-write and reasoning shares)
* - Daily costs (if num_requests_per_day provided)
* - Monthly costs (if num_requests_per_month provided)
*
@ -3331,7 +3334,9 @@ export interface paths {
* {
* "model": "gpt-4",
* "input_tokens": 1000,
* "cache_read_input_tokens": 800,
* "output_tokens": 500,
* "reasoning_tokens": 200,
* "num_requests_per_day": 100,
* "num_requests_per_month": 3000
* }
@ -26468,6 +26473,18 @@ export interface components {
* @description Request body for /cost/estimate endpoint.
*/
CostEstimateRequest: {
/**
* Cache Creation Input Tokens
* @description Input tokens written to the prompt cache; counted within input_tokens
* @default 0
*/
cache_creation_input_tokens: number;
/**
* Cache Read Input Tokens
* @description Input tokens read from the prompt cache; counted within input_tokens
* @default 0
*/
cache_read_input_tokens: number;
/**
* Input Tokens
* @description Expected input tokens per request
@ -26493,17 +26510,65 @@ export interface components {
* @description Expected output tokens per request
*/
output_tokens: number;
/**
* Reasoning Tokens
* @description Reasoning tokens the model emits; counted within output_tokens
* @default 0
*/
reasoning_tokens: number;
};
/**
* CostEstimateResponse
* @description Response body for /cost/estimate endpoint.
*/
CostEstimateResponse: {
/**
* Cache Creation Cost Per Request
* @description Cache-write share of input_cost_per_request
* @default 0
*/
cache_creation_cost_per_request: number;
/**
* Cache Creation Input Token Cost
* @description Rate billed per cache-write token
*/
cache_creation_input_token_cost?: number | null;
/**
* Cache Creation Input Tokens
* @default 0
*/
cache_creation_input_tokens: number;
/**
* Cache Read Cost Per Request
* @description Cache-read share of input_cost_per_request
* @default 0
*/
cache_read_cost_per_request: number;
/**
* Cache Read Input Token Cost
* @description Rate billed per cache-read token
*/
cache_read_input_token_cost?: number | null;
/**
* Cache Read Input Tokens
* @default 0
*/
cache_read_input_tokens: number;
/**
* Cost Per Request
* @description Total cost per request (includes margin)
*/
cost_per_request: number;
/**
* Daily Cache Creation Cost
* @description Cache-write share of daily_input_cost
*/
daily_cache_creation_cost?: number | null;
/**
* Daily Cache Read Cost
* @description Cache-read share of daily_input_cost
*/
daily_cache_read_cost?: number | null;
/**
* Daily Cost
* @description Total daily cost (includes margin)
@ -26524,12 +26589,20 @@ export interface components {
* @description Daily output token cost
*/
daily_output_cost?: number | null;
/**
* Daily Reasoning Cost
* @description Reasoning share of daily_output_cost
*/
daily_reasoning_cost?: number | null;
/**
* Input Cost Per Request
* @description Input token cost per request (before margin)
*/
input_cost_per_request: number;
/** Input Cost Per Token */
/**
* Input Cost Per Token
* @description Rate billed per input token
*/
input_cost_per_token?: number | null;
/** Input Tokens */
input_tokens: number;
@ -26541,6 +26614,16 @@ export interface components {
margin_cost_per_request: number;
/** Model */
model: string;
/**
* Monthly Cache Creation Cost
* @description Cache-write share of monthly_input_cost
*/
monthly_cache_creation_cost?: number | null;
/**
* Monthly Cache Read Cost
* @description Cache-read share of monthly_input_cost
*/
monthly_cache_read_cost?: number | null;
/**
* Monthly Cost
* @description Total monthly cost (includes margin)
@ -26561,21 +26644,45 @@ export interface components {
* @description Monthly output token cost
*/
monthly_output_cost?: number | null;
/**
* Monthly Reasoning Cost
* @description Reasoning share of monthly_output_cost
*/
monthly_reasoning_cost?: number | null;
/** Num Requests Per Day */
num_requests_per_day?: number | null;
/** Num Requests Per Month */
num_requests_per_month?: number | null;
/**
* Output Cost Per Reasoning Token
* @description Rate billed per reasoning token
*/
output_cost_per_reasoning_token?: number | null;
/**
* Output Cost Per Request
* @description Output token cost per request (before margin)
*/
output_cost_per_request: number;
/** Output Cost Per Token */
/**
* Output Cost Per Token
* @description Rate billed per output token
*/
output_cost_per_token?: number | null;
/** Output Tokens */
output_tokens: number;
/** Provider */
provider?: string | null;
/**
* Reasoning Cost Per Request
* @description Reasoning share of output_cost_per_request
* @default 0
*/
reasoning_cost_per_request: number;
/**
* Reasoning Tokens
* @default 0
*/
reasoning_tokens: number;
};
/** CreateCredentialItem */
CreateCredentialItem: {