mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
fix(proxy): report the billed token rates in /cost/estimate
The rate fields reported base cost-map prices while the cost lines were billed at the token tier, off-peak window and regional multipliers the calculator picks for the request, so a line did not always equal tokens times its reported rate. get_billed_token_rates now resolves the rates once, the token-type breakdown and the endpoint both read from it, and a tiered-model test asserts every line equals its token count times the rate reported next to it Claude-Session: https://claude.ai/code/session_011Tn3657NkV6ojLqewL64Kb
This commit is contained in:
ryan-crabbe-berri
parent
c84131b81a
commit
43a02e2dbc
6 changed files with 258 additions and 128 deletions
|
|
@ -1329,16 +1329,118 @@ def _cache_token_counts(usage: Usage) -> tuple[int, int, CacheCreationTokenDetai
|
|||
)
|
||||
|
||||
|
||||
def _custom_pricing_token_type_breakdown(usage: Usage, custom_cost_per_token: CostPerToken) -> TokenTypeCostBreakdown:
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BilledTokenRates:
|
||||
"""Per-token rates one request's usage bills at, after token tiers, off-peak windows and the
|
||||
regional multipliers the totals apply, so each cost line equals its token count times its rate."""
|
||||
|
||||
input_cost_per_token: float
|
||||
output_cost_per_token: float
|
||||
cache_read_input_token_cost: float
|
||||
cache_creation_input_token_cost: float
|
||||
cache_creation_input_token_cost_above_1hr: float
|
||||
output_cost_per_reasoning_token: float
|
||||
|
||||
def scaled(self, multiplier: float) -> "BilledTokenRates":
|
||||
if multiplier == 1.0:
|
||||
return self
|
||||
return BilledTokenRates(
|
||||
input_cost_per_token=self.input_cost_per_token * multiplier,
|
||||
output_cost_per_token=self.output_cost_per_token * multiplier,
|
||||
cache_read_input_token_cost=self.cache_read_input_token_cost * multiplier,
|
||||
cache_creation_input_token_cost=self.cache_creation_input_token_cost * multiplier,
|
||||
cache_creation_input_token_cost_above_1hr=self.cache_creation_input_token_cost_above_1hr * multiplier,
|
||||
output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier,
|
||||
)
|
||||
|
||||
|
||||
def _custom_pricing_rates(custom_cost_per_token: CostPerToken) -> BilledTokenRates:
|
||||
"""Flat custom pricing has no tiers, uplifts or reasoning rate: cache tokens bill at the configured
|
||||
cache rates (else the input rate) and reasoning at the output rate, as _cost_per_token_custom_pricing_helper does."""
|
||||
input_rate: Final = custom_cost_per_token["input_cost_per_token"]
|
||||
cache_read_tokens, cache_creation_tokens, _ = _cache_token_counts(usage)
|
||||
return TokenTypeCostBreakdown(
|
||||
reasoning_cost=float(_reasoning_token_count(usage)) * custom_cost_per_token["output_cost_per_token"],
|
||||
cache_read_cost=float(cache_read_tokens) * custom_cost_per_token.get("cache_read_input_token_cost", input_rate),
|
||||
cache_creation_cost=float(cache_creation_tokens)
|
||||
* custom_cost_per_token.get("cache_creation_input_token_cost", input_rate),
|
||||
output_rate: Final = custom_cost_per_token["output_cost_per_token"]
|
||||
cache_creation_rate: Final = custom_cost_per_token.get("cache_creation_input_token_cost", input_rate)
|
||||
return BilledTokenRates(
|
||||
input_cost_per_token=input_rate,
|
||||
output_cost_per_token=output_rate,
|
||||
cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate),
|
||||
cache_creation_input_token_cost=cache_creation_rate,
|
||||
cache_creation_input_token_cost_above_1hr=cache_creation_rate,
|
||||
output_cost_per_reasoning_token=output_rate,
|
||||
)
|
||||
|
||||
|
||||
def _cost_map_billed_rates(
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
custom_llm_provider: str | None,
|
||||
service_tier: str | None,
|
||||
data_residency: str | None,
|
||||
vertex_location: str | None,
|
||||
current_time: datetime | None,
|
||||
) -> BilledTokenRates:
|
||||
billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc)
|
||||
(
|
||||
prompt_base_cost,
|
||||
completion_base_cost,
|
||||
cache_creation_cost_rate,
|
||||
cache_creation_cost_above_1hr_rate,
|
||||
cache_read_cost_rate,
|
||||
) = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
current_time=billing_time,
|
||||
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
|
||||
)
|
||||
reasoning_rate: Final = _resolve_billed_reasoning_rate(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
completion_base_cost=completion_base_cost,
|
||||
current_time=billing_time,
|
||||
)
|
||||
multiplier: Final = (
|
||||
_get_regional_uplift_multiplier(model_info, data_residency)
|
||||
* get_vertex_regional_endpoint_uplift(model_info, vertex_location)
|
||||
* get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
|
||||
)
|
||||
return BilledTokenRates(
|
||||
input_cost_per_token=prompt_base_cost,
|
||||
output_cost_per_token=completion_base_cost,
|
||||
cache_read_input_token_cost=cache_read_cost_rate,
|
||||
cache_creation_input_token_cost=cache_creation_cost_rate,
|
||||
cache_creation_input_token_cost_above_1hr=cache_creation_cost_above_1hr_rate,
|
||||
output_cost_per_reasoning_token=reasoning_rate,
|
||||
).scaled(multiplier)
|
||||
|
||||
|
||||
def get_billed_token_rates(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
usage: Usage,
|
||||
service_tier: str | None = None,
|
||||
data_residency: str | None = None,
|
||||
vertex_location: str | None = None,
|
||||
current_time: datetime | None = None,
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
) -> BilledTokenRates | None:
|
||||
"""Rates the cost calculator bills ``usage`` at, resolved exactly as the totals and the token-type
|
||||
breakdown resolve them. None when the model's pricing cannot be resolved."""
|
||||
if custom_cost_per_token is not None:
|
||||
return _custom_pricing_rates(custom_cost_per_token)
|
||||
try:
|
||||
model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception:
|
||||
return None
|
||||
return _cost_map_billed_rates(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
current_time=current_time,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1360,77 +1462,39 @@ def get_token_type_cost_breakdown(
|
|||
cost calculators bypass ``generic_cost_per_token``, because cache tokens always
|
||||
land on ``prompt_tokens_details`` (via the Usage constructor and provider
|
||||
transformations) and reasoning tokens on ``completion_tokens_details``. It reuses
|
||||
the same rate-resolution primitives as the total-cost path so the breakdown can
|
||||
never drift from the totals. A deployment billed by ``custom_cost_per_token`` is
|
||||
priced from those flat rates instead of the cost map, for the same reason.
|
||||
the same rate resolution as the total-cost path (``get_billed_token_rates``) so the
|
||||
breakdown can never drift from the totals. A deployment billed by
|
||||
``custom_cost_per_token`` is priced from those flat rates instead of the cost map and,
|
||||
like its totals, bills cache writes flat rather than by their 5m/1h split.
|
||||
Returns zeros (never raises) when the model or its pricing cannot be resolved.
|
||||
"""
|
||||
if custom_cost_per_token is not None:
|
||||
return _custom_pricing_token_type_breakdown(usage=usage, custom_cost_per_token=custom_cost_per_token)
|
||||
|
||||
try:
|
||||
model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception:
|
||||
rates: Final = get_billed_token_rates(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
current_time=current_time,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
)
|
||||
if rates is None:
|
||||
return TokenTypeCostBreakdown(0.0, 0.0, 0.0)
|
||||
|
||||
billing_time: Final = current_time if current_time is not None else datetime.now(timezone.utc)
|
||||
(
|
||||
_prompt_base_cost,
|
||||
completion_base_cost,
|
||||
cache_creation_cost_rate,
|
||||
cache_creation_cost_above_1hr_rate,
|
||||
cache_read_cost_rate,
|
||||
) = _get_token_base_cost(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
current_time=billing_time,
|
||||
threshold_is_inclusive=_uses_inclusive_token_thresholds(custom_llm_provider),
|
||||
)
|
||||
|
||||
reasoning_rate: Final = _resolve_billed_reasoning_rate(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
completion_base_cost=completion_base_cost,
|
||||
current_time=billing_time,
|
||||
)
|
||||
reasoning_cost = float(_reasoning_token_count(usage)) * reasoning_rate
|
||||
|
||||
cache_read_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts(usage)
|
||||
cache_read_cost = float(cache_read_tokens) * cache_read_cost_rate
|
||||
cache_creation_cost = calculate_cache_writing_cost(
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=cache_creation_token_details,
|
||||
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr_rate,
|
||||
cache_creation_cost=cache_creation_cost_rate,
|
||||
cache_creation_cost: Final = (
|
||||
float(cache_creation_tokens) * rates.cache_creation_input_token_cost
|
||||
if custom_cost_per_token is not None
|
||||
else calculate_cache_writing_cost(
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cache_creation_token_details=cache_creation_token_details,
|
||||
cache_creation_cost_above_1hr=rates.cache_creation_input_token_cost_above_1hr,
|
||||
cache_creation_cost=rates.cache_creation_input_token_cost,
|
||||
)
|
||||
)
|
||||
|
||||
# Apply the same flat regional-processing uplift the totals get, so per-type
|
||||
# costs stay reconciled with input_cost/output_cost for regionalized OpenAI hosts.
|
||||
uplift: Final = _get_regional_uplift_multiplier(model_info, data_residency)
|
||||
if uplift != 1.0:
|
||||
reasoning_cost *= uplift
|
||||
cache_read_cost *= uplift
|
||||
cache_creation_cost *= uplift
|
||||
|
||||
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
|
||||
if vertex_uplift != 1.0:
|
||||
reasoning_cost *= vertex_uplift
|
||||
cache_read_cost *= vertex_uplift
|
||||
cache_creation_cost *= vertex_uplift
|
||||
|
||||
# Mirror the provider-specific geo uplift (e.g. Anthropic us: 1.1) the totals
|
||||
# apply, so cache and reasoning line items stay reconciled with them.
|
||||
geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
|
||||
if geo_multiplier != 1.0:
|
||||
reasoning_cost *= geo_multiplier
|
||||
cache_read_cost *= geo_multiplier
|
||||
cache_creation_cost *= geo_multiplier
|
||||
|
||||
return TokenTypeCostBreakdown(
|
||||
reasoning_cost=reasoning_cost,
|
||||
cache_read_cost=cache_read_cost,
|
||||
reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token,
|
||||
cache_read_cost=float(cache_read_tokens) * rates.cache_read_input_token_cost,
|
||||
cache_creation_cost=cache_creation_cost,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -5197,9 +5197,9 @@ class CostEstimateResponse(LiteLLMPydanticObjectBase):
|
|||
default=None, description="Cache-write share of monthly_input_cost"
|
||||
)
|
||||
monthly_reasoning_cost: float | None = Field(default=None, description="Reasoning share of monthly_output_cost")
|
||||
# Pricing info
|
||||
input_cost_per_token: float | None = None
|
||||
output_cost_per_token: float | None = None
|
||||
# Pricing info: the rates this request's usage bills at, after token tiers and regional multipliers
|
||||
input_cost_per_token: float | None = Field(default=None, description="Rate billed per input token")
|
||||
output_cost_per_token: float | None = Field(default=None, description="Rate billed per output token")
|
||||
cache_read_input_token_cost: float | None = Field(default=None, description="Rate billed per cache-read token")
|
||||
cache_creation_input_token_cost: float | None = Field(default=None, description="Rate billed per cache-write token")
|
||||
output_cost_per_reasoning_token: float | None = Field(default=None, description="Rate billed per reasoning token")
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from pydantic import BaseModel
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.cost_calculator import completion_cost
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import get_billed_token_rates
|
||||
from litellm.proxy._types import (
|
||||
CommonProxyErrors,
|
||||
CostEstimateRequest,
|
||||
|
|
@ -170,44 +171,6 @@ def _cost_lines(cost_per_request: float, cost_breakdown: CostBreakdown | None) -
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EffectiveTokenRates:
|
||||
input_cost_per_token: float | None
|
||||
output_cost_per_token: float | None
|
||||
cache_read_input_token_cost: float | None
|
||||
cache_creation_input_token_cost: float | None
|
||||
output_cost_per_reasoning_token: float | None
|
||||
|
||||
|
||||
def _custom_token_rates(custom_cost_per_token: CostPerToken) -> EffectiveTokenRates:
|
||||
input_rate: Final = custom_cost_per_token["input_cost_per_token"]
|
||||
output_rate: Final = custom_cost_per_token["output_cost_per_token"]
|
||||
return EffectiveTokenRates(
|
||||
input_cost_per_token=input_rate,
|
||||
output_cost_per_token=output_rate,
|
||||
cache_read_input_token_cost=custom_cost_per_token.get("cache_read_input_token_cost", input_rate),
|
||||
cache_creation_input_token_cost=custom_cost_per_token.get("cache_creation_input_token_cost", input_rate),
|
||||
output_cost_per_reasoning_token=output_rate,
|
||||
)
|
||||
|
||||
|
||||
def _cost_map_token_rates(model_info: ModelInfo | None) -> EffectiveTokenRates:
|
||||
"""Base rates the cost calculator bills flat usage at: a cost-map model without a cache price
|
||||
bills cache tokens at zero, and one without a reasoning price bills reasoning at the output rate."""
|
||||
if model_info is None:
|
||||
return EffectiveTokenRates(None, None, None, None, None)
|
||||
sources: Final = (model_info,)
|
||||
output_rate: Final = _configured_price("output_cost_per_token", sources)
|
||||
reasoning_rate: Final = _configured_price("output_cost_per_reasoning_token", sources)
|
||||
return EffectiveTokenRates(
|
||||
input_cost_per_token=_configured_price("input_cost_per_token", sources),
|
||||
output_cost_per_token=output_rate,
|
||||
cache_read_input_token_cost=_configured_price("cache_read_input_token_cost", sources) or 0.0,
|
||||
cache_creation_input_token_cost=_configured_price("cache_creation_input_token_cost", sources) or 0.0,
|
||||
output_cost_per_reasoning_token=output_rate if reasoning_rate is None else reasoning_rate,
|
||||
)
|
||||
|
||||
|
||||
def _usage_for_estimate(request: CostEstimateRequest) -> Usage:
|
||||
cache_tokens: Final = request.cache_read_input_tokens + request.cache_creation_input_tokens
|
||||
return Usage(
|
||||
|
|
@ -654,7 +617,8 @@ async def estimate_cost(
|
|||
|
||||
verbose_proxy_logger.debug("Cost estimate: request.model='%s' resolved to '%s'", request.model, resolved_model)
|
||||
|
||||
mock_response: Final = ModelResponse(model=resolved_model, usage=_usage_for_estimate(request))
|
||||
usage: Final = _usage_for_estimate(request)
|
||||
mock_response: Final = ModelResponse(model=resolved_model, usage=usage)
|
||||
|
||||
# Create a logging object to capture cost breakdown
|
||||
litellm_logging_obj: Final = LiteLLMLoggingObj(
|
||||
|
|
@ -688,12 +652,13 @@ async def estimate_cost(
|
|||
daily: Final = per_request.times(request.num_requests_per_day)
|
||||
monthly: Final = per_request.times(request.num_requests_per_month)
|
||||
|
||||
model_info: Final = _lookup_model_info(resolved_model)
|
||||
rates: Final = (
|
||||
_custom_token_rates(resolved.custom_cost_per_token)
|
||||
if resolved.custom_cost_per_token is not None
|
||||
else _cost_map_token_rates(model_info)
|
||||
rates: Final = get_billed_token_rates(
|
||||
model=resolved_model,
|
||||
custom_llm_provider=resolved_provider,
|
||||
usage=usage,
|
||||
custom_cost_per_token=resolved.custom_cost_per_token,
|
||||
)
|
||||
model_info: Final = _lookup_model_info(resolved_model)
|
||||
mapped_provider: Final = model_info.get("litellm_provider") if model_info is not None else None
|
||||
custom_llm_provider: Final = mapped_provider if mapped_provider is not None else resolved_provider
|
||||
|
||||
|
|
@ -727,10 +692,10 @@ async def estimate_cost(
|
|||
monthly_cache_read_cost=monthly.cache_read_cost if monthly is not None else None,
|
||||
monthly_cache_creation_cost=monthly.cache_creation_cost if monthly is not None else None,
|
||||
monthly_reasoning_cost=monthly.reasoning_cost if monthly is not None else None,
|
||||
input_cost_per_token=rates.input_cost_per_token,
|
||||
output_cost_per_token=rates.output_cost_per_token,
|
||||
cache_read_input_token_cost=rates.cache_read_input_token_cost,
|
||||
cache_creation_input_token_cost=rates.cache_creation_input_token_cost,
|
||||
output_cost_per_reasoning_token=rates.output_cost_per_reasoning_token,
|
||||
input_cost_per_token=rates.input_cost_per_token if rates is not None else None,
|
||||
output_cost_per_token=rates.output_cost_per_token if rates is not None else None,
|
||||
cache_read_input_token_cost=rates.cache_read_input_token_cost if rates is not None else None,
|
||||
cache_creation_input_token_cost=rates.cache_creation_input_token_cost if rates is not None else None,
|
||||
output_cost_per_reasoning_token=rates.output_cost_per_reasoning_token if rates is not None else None,
|
||||
provider=custom_llm_provider,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
BilledTokenRates,
|
||||
CostCalculatorUtils,
|
||||
PromptTokensDetailsResult,
|
||||
TokenRates,
|
||||
|
|
@ -38,6 +39,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
apply_off_peak_pricing,
|
||||
calculate_cache_writing_cost,
|
||||
generic_cost_per_token,
|
||||
get_billed_token_rates,
|
||||
get_token_type_cost_breakdown,
|
||||
)
|
||||
from litellm.types.utils import CacheCreationTokenDetails, Usage
|
||||
|
|
@ -3938,6 +3940,53 @@ def test_token_type_cost_breakdown_reconciles_with_custom_pricing_totals():
|
|||
assert 300 * 2e-6 + breakdown.reasoning_cost == pytest.approx(completion_cost)
|
||||
|
||||
|
||||
def test_billed_token_rates_follow_the_token_tier_the_breakdown_bills_at(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"tiered-cache-model",
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"cache_read_input_token_cost": 3e-7,
|
||||
"cache_creation_input_token_cost": 3.75e-6,
|
||||
"input_cost_per_token_above_200k_tokens": 6e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 3e-5,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-7,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-6,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
},
|
||||
)
|
||||
usage = Usage(
|
||||
prompt_tokens=250_000,
|
||||
completion_tokens=1_000,
|
||||
total_tokens=251_000,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200_000, cache_creation_tokens=10_000),
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=200),
|
||||
)
|
||||
|
||||
rates = get_billed_token_rates(model="tiered-cache-model", custom_llm_provider="openai", usage=usage)
|
||||
breakdown = get_token_type_cost_breakdown(model="tiered-cache-model", custom_llm_provider="openai", usage=usage)
|
||||
|
||||
assert rates == BilledTokenRates(
|
||||
input_cost_per_token=6e-6,
|
||||
output_cost_per_token=3e-5,
|
||||
cache_read_input_token_cost=6e-7,
|
||||
cache_creation_input_token_cost=7.5e-6,
|
||||
cache_creation_input_token_cost_above_1hr=0.0,
|
||||
output_cost_per_reasoning_token=3e-5,
|
||||
)
|
||||
assert breakdown.cache_read_cost == pytest.approx(200_000 * rates.cache_read_input_token_cost)
|
||||
assert breakdown.cache_creation_cost == pytest.approx(10_000 * rates.cache_creation_input_token_cost)
|
||||
assert breakdown.reasoning_cost == pytest.approx(200 * rates.output_cost_per_reasoning_token)
|
||||
|
||||
|
||||
def test_billed_token_rates_are_none_for_an_unpriced_model():
|
||||
usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
|
||||
|
||||
assert get_billed_token_rates(model="no-such-model-anywhere", custom_llm_provider="openai", usage=usage) is None
|
||||
|
||||
|
||||
def test_token_type_cost_breakdown_zero_without_special_tokens(_local_model_cost_map):
|
||||
|
||||
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
|
||||
|
|
|
|||
|
|
@ -813,9 +813,7 @@ async def _estimate(mock_router: MagicMock | None, model: str = AN_ALIAS, **over
|
|||
|
||||
request = CostEstimateRequest(
|
||||
model=model,
|
||||
input_tokens=INPUT_TOKENS,
|
||||
output_tokens=OUTPUT_TOKENS,
|
||||
**overrides,
|
||||
**{"input_tokens": INPUT_TOKENS, "output_tokens": OUTPUT_TOKENS, **overrides},
|
||||
)
|
||||
with patch( # test-quality-ok: proxy_server module global is the endpoint's only injection point
|
||||
"litellm.proxy.proxy_server.llm_router", mock_router
|
||||
|
|
@ -1062,6 +1060,54 @@ class TestEstimateCostCacheAndReasoningTokens:
|
|||
assert response.cache_read_input_token_cost == pytest.approx(5e-7)
|
||||
assert response.cache_creation_input_token_cost == pytest.approx(6.25e-6)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_tiered_model_reports_the_rates_its_lines_were_billed_at(self, monkeypatch):
|
||||
"""Above a token tier the calculator bills every line at the tier's rate, so the reported
|
||||
rates must be the tier's too: each line equals its token count times the rate next to it."""
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
A_MAPPED_MODEL,
|
||||
{
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"cache_read_input_token_cost": 3e-7,
|
||||
"cache_creation_input_token_cost": 3.75e-6,
|
||||
"input_cost_per_token_above_200k_tokens": 6e-6,
|
||||
"output_cost_per_token_above_200k_tokens": 3e-5,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-7,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-6,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
},
|
||||
)
|
||||
|
||||
response = await _estimate(
|
||||
None,
|
||||
model=A_MAPPED_MODEL,
|
||||
input_tokens=250_000,
|
||||
cache_read_input_tokens=200_000,
|
||||
cache_creation_input_tokens=10_000,
|
||||
output_tokens=1_000,
|
||||
reasoning_tokens=200,
|
||||
)
|
||||
|
||||
assert response.input_cost_per_token == pytest.approx(6e-6)
|
||||
assert response.output_cost_per_token == pytest.approx(3e-5)
|
||||
assert response.cache_read_input_token_cost == pytest.approx(6e-7)
|
||||
assert response.cache_creation_input_token_cost == pytest.approx(7.5e-6)
|
||||
assert response.output_cost_per_reasoning_token == pytest.approx(3e-5)
|
||||
assert response.cache_read_cost_per_request == pytest.approx(200_000 * response.cache_read_input_token_cost)
|
||||
assert response.cache_creation_cost_per_request == pytest.approx(
|
||||
10_000 * response.cache_creation_input_token_cost
|
||||
)
|
||||
assert response.reasoning_cost_per_request == pytest.approx(200 * response.output_cost_per_reasoning_token)
|
||||
assert response.input_cost_per_request == pytest.approx(
|
||||
40_000 * response.input_cost_per_token
|
||||
+ response.cache_read_cost_per_request
|
||||
+ response.cache_creation_cost_per_request
|
||||
)
|
||||
assert response.output_cost_per_request == pytest.approx(1_000 * response.output_cost_per_token)
|
||||
|
||||
|
||||
class TestCostEstimateRequestTokenSubsets:
|
||||
def test_cache_tokens_beyond_the_input_tokens_are_rejected(self):
|
||||
|
|
|
|||
10
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
10
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -26523,7 +26523,10 @@ export interface components {
|
|||
* @description Input token cost per request (before margin)
|
||||
*/
|
||||
input_cost_per_request: number;
|
||||
/** Input Cost Per Token */
|
||||
/**
|
||||
* Input Cost Per Token
|
||||
* @description Rate billed per input token
|
||||
*/
|
||||
input_cost_per_token?: number | null;
|
||||
/** Input Tokens */
|
||||
input_tokens: number;
|
||||
|
|
@ -26584,7 +26587,10 @@ export interface components {
|
|||
* @description Output token cost per request (before margin)
|
||||
*/
|
||||
output_cost_per_request: number;
|
||||
/** Output Cost Per Token */
|
||||
/**
|
||||
* Output Cost Per Token
|
||||
* @description Rate billed per output token
|
||||
*/
|
||||
output_cost_per_token?: number | null;
|
||||
/** Output Tokens */
|
||||
output_tokens: number;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue