refactor(cost): move SDK token pricing to native engine

This commit is contained in:
Devin AI 2026-10-26 11:05:58 +00:00
parent 7479e494b0
commit ce5ded9bf2
33 changed files with 2566 additions and 2624 deletions

View file

@ -15,6 +15,12 @@ class Reloaded:
models_count: int
def is_priced(model: str, provider: str | None = None) -> bool:
from litellm._catalog import _read
return _read.is_priced(model, provider)
def reload_source(url: str) -> Source:
if not url or is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
return Bundled()

View file

@ -7,7 +7,7 @@ from functools import wraps
from itertools import chain
from typing import TYPE_CHECKING, Final, Literal, ParamSpec, TypeVar, overload
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
from litellm._catalog import _install
from litellm.rust_bridge import _native
@ -154,6 +154,27 @@ def model_info(key: str) -> dict[str, object]:
return {name: raw[name] for name in result.fields}
def is_priced(model: str, provider: str | None = None) -> bool:
resolved: Final = resolve(model, provider)
if not isinstance(resolved, _native.ResolvedModel):
return False
try:
row: Final = _ROW.validate_python(
dict[str, object].get(
_install._live_rows(), # pyright: ignore[reportPrivateUsage] # presence uses the selected live row
resolved.key,
)
)
except ValidationError:
return False
contextual_row: Final = row if "litellm_provider" in row else {"litellm_provider": provider or "custom", **row}
match _native.has_token_prices(contextual_row, policy="sdk_tokens"):
case bool() as present:
return present
case _native.RowErrors():
return False
def supports(fields: Mapping[str, object], capability: str) -> bool | None:
value: Final = fields.get(capability)
return None if value is None else value is True

23
litellm/_cost/__init__.py Normal file
View file

@ -0,0 +1,23 @@
from litellm._cost._price import (
billing_context,
cache_read_meter,
cache_write_quantities,
evaluate,
price,
request_model_info,
request_prices,
require_priced,
token_usage,
)
__all__ = (
"billing_context",
"cache_read_meter",
"cache_write_quantities",
"evaluate",
"price",
"request_model_info",
"request_prices",
"require_priced",
"token_usage",
)

252
litellm/_cost/_price.py Normal file
View file

@ -0,0 +1,252 @@
from collections.abc import Mapping, Sequence
from datetime import datetime, timezone
from typing import (
Final,
Literal,
cast, # noqa: TID251 # validated request prices extend the read-only public ModelInfo shape
)
from pydantic import TypeAdapter
from litellm.exceptions import ModelNotMappedError
from litellm.rust_bridge import _native
from litellm.types.utils import CacheCreationTokenDetails, CustomPricingLiteLLMParams, ModelInfo, Usage
_REQUEST_ROW: Final = TypeAdapter(dict[str, object])
_REQUEST_PRICE_KEYS: Final = frozenset(CustomPricingLiteLLMParams.model_fields)
def _request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]:
info: Final = kwargs.get("model_info")
if not isinstance(info, Mapping):
return {}
fields: Final = _REQUEST_ROW.validate_python(info)
return {
key: value
for key, value in fields.items()
if key in ("mode", "supports_prompt_caching", "max_tokens") and value is not None
}
def request_prices(kwargs: Mapping[str, object]) -> Mapping[str, object] | None:
has_tokens: Final = (
kwargs.get("input_cost_per_token") is not None and kwargs.get("output_cost_per_token") is not None
)
has_seconds: Final = any(
kwargs.get(key) is not None for key in ("cost_per_second", "input_cost_per_second", "output_cost_per_second")
)
if not has_tokens and not has_seconds:
return None
prices: Final = CustomPricingLiteLLMParams.model_validate(
{key: value for key, value in kwargs.items() if key in _REQUEST_PRICE_KEYS and value is not None}
)
return {
**_request_metadata(kwargs),
**_REQUEST_ROW.validate_python(prices.model_dump(exclude_none=True, exclude_unset=True)),
}
def request_model_info(model: str, provider: str | None, overrides: Mapping[str, object]) -> ModelInfo:
from litellm.utils import get_model_info
def live_info() -> ModelInfo:
try:
return get_model_info(model=model, custom_llm_provider=provider)
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unknown custom model
return {
"key": model,
"litellm_provider": provider or "custom",
"mode": "chat",
"input_cost_per_token": None,
"output_cost_per_token": None,
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
"supported_openai_params": None,
}
return cast( # cast-ok: validated pricing keys preserve extra request meters
ModelInfo, {**live_info(), **overrides}
)
def price(
row: Mapping[str, object],
usage: _native.UsageInput,
when: _native.WhenInput,
*,
factors: Sequence[float] = (),
cache_read_default_ratio: float | None = None,
directional_token_threshold: int | None = None,
) -> _native.Priced | _native.Unpriced | _native.RowErrors:
if not factors:
return _native.price(
dict(row),
usage,
when,
policy="sdk_tokens",
cache_read_default_ratio=cache_read_default_ratio,
directional_token_threshold=directional_token_threshold,
)
result: Final = evaluate(
row,
usage,
when,
factors=factors,
cache_read_default_ratio=cache_read_default_ratio,
directional_token_threshold=directional_token_threshold,
)
return result if isinstance(result, _native.RowErrors) else result.pricing
def evaluate(
row: Mapping[str, object],
usage: _native.UsageInput,
when: _native.WhenInput,
*,
project: Sequence[str] = (),
factors: Sequence[float] = (),
cache_read_default_ratio: float | None = None,
directional_token_threshold: int | None = None,
) -> _native.PriceEvaluation | _native.RowErrors:
return _native.price_with_rates(
dict(row),
usage,
when,
policy="sdk_tokens",
project=project,
factors=factors,
cache_read_default_ratio=cache_read_default_ratio,
directional_token_threshold=directional_token_threshold,
)
def require_priced(result: _native.Priced | _native.Unpriced | _native.RowErrors, model: str) -> _native.Priced:
if isinstance(result, _native.Priced):
return result
detail: Final = (
result.missing
if isinstance(result, _native.Unpriced)
else tuple(f"{error.field}: {error.message}" for error in result.errors)
)
raise ModelNotMappedError(f"No valid pricing for model={model}: {detail}")
def billing_context(
current_time: datetime,
service_tier: str | None = None,
channel: Literal["online", "batch", "decisions"] = "online",
) -> _native.WhenInput:
normalized: Final = service_tier.strip().lower() if service_tier else "standard"
tier: Final = (
"priority"
if normalized == "fast"
else normalized
if normalized in ("priority", "flex", "ultrafast", "balanced")
else "standard"
)
utc_time: Final = current_time if current_time.tzinfo is not None else current_time.replace(tzinfo=timezone.utc)
return _native.WhenInput(int(utc_time.timestamp()), tier=tier, channel=channel)
def cache_read_meter(usage: Usage) -> Literal["cache_read_input_token_cost", "explicit_cache_read_input_token_cost"]:
return (
"explicit_cache_read_input_token_cost"
if getattr(usage.prompt_tokens_details, "cache_type", None) == "ephemeral"
else "cache_read_input_token_cost"
)
def cache_write_quantities(cache_creation: int, ttl: CacheCreationTokenDetails | None) -> Mapping[str, int]:
short_writes: Final = ((ttl.ephemeral_5m_input_tokens or 0) + (ttl.ephemeral_30m_input_tokens or 0)) if ttl else 0
writes: Final = {
"cache_creation_input_token_cost_above_1hr": (ttl.ephemeral_1h_input_tokens or 0) if ttl else 0,
"cache_creation_input_token_cost_above_12hr": (ttl.ephemeral_12h_input_tokens or 0) if ttl else 0,
}
return {
"cache_creation_input_token_cost": max(cache_creation - short_writes - sum(writes.values()), 0) + short_writes,
**writes,
}
def _prompt_quantities(usage: Usage, row: Mapping[str, object]) -> Mapping[str, int]:
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
if usage.prompt_tokens_details is None:
return {"input_cost_per_token": usage.prompt_tokens}
details: Final = parse_prompt_tokens_details(usage)
cache_hit: Final = details["cache_hit_tokens"]
cache_creation: Final = details["cache_creation_tokens"]
total_details: Final = sum(
(
details["text_tokens"],
cache_hit,
cache_creation,
details["audio_tokens"],
details["image_tokens"],
details["video_tokens"],
)
)
overlaps: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens
uncached: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0)
audio: Final = min(details["audio_tokens"], uncached) if overlaps else details["audio_tokens"]
image: Final = min(details["image_tokens"], uncached - audio) if overlaps else details["image_tokens"]
video: Final = min(details["video_tokens"], uncached - audio - image) if overlaps else details["video_tokens"]
remainder: Final = max(uncached - audio - image - video, 0)
text: Final = (
remainder
if overlaps or (details["text_tokens"] == 0 and details["image_count"] == 0)
else details["text_tokens"]
)
return {
"input_cost_per_token": 0
if details["character_count"] and row.get("input_cost_per_character") is not None
else text,
"input_cost_per_audio_token": 0
if details["audio_length_seconds"] and row.get("input_cost_per_audio_per_second") is not None
else audio,
"input_cost_per_image_token": 0
if details["image_count"] and row.get("input_cost_per_image") is not None
else image,
"input_cost_per_video_token": 0
if details["video_length_seconds"] and row.get("input_cost_per_video_per_second") is not None
else video,
cache_read_meter(usage): cache_hit - details["cache_hit_audio_tokens"],
"cache_read_input_audio_token_cost": details["cache_hit_audio_tokens"],
**cache_write_quantities(cache_creation, details["cache_creation_token_details"]),
}
def _completion_quantities(usage: Usage, *, batch: bool = False) -> Mapping[str, int]:
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_completion_tokens_details
if usage.completion_tokens_details is None:
return {"output_cost_per_token": usage.completion_tokens}
details: Final = parse_completion_tokens_details(usage)
image_tokens: Final = min(details["image_tokens"], usage.completion_tokens) if batch else details["image_tokens"]
modalities: Final = {
"output_cost_per_audio_token": details["audio_tokens"],
"output_cost_per_reasoning_token": details["reasoning_tokens"],
"output_cost_per_image_token": image_tokens,
"output_cost_per_video_token": details["video_tokens"],
}
text: Final = details["text_tokens"] or max(usage.completion_tokens - sum(modalities.values()), 0)
return {"output_cost_per_token": text, **modalities}
def token_usage(
usage: Usage,
row: Mapping[str, object],
*,
inclusive_threshold: bool = False,
batch: bool = False,
direction: Literal["input", "output"] | None = None,
) -> _native.UsageInput:
return _native.UsageInput(
{
**(_prompt_quantities(usage, row) if direction != "output" else {}),
**(_completion_quantities(usage, batch=batch) if direction != "input" else {}),
},
prompt_tokens=usage.prompt_tokens,
inclusive_threshold=inclusive_threshold,
)

View file

@ -16,6 +16,8 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
import litellm
import litellm._logging
from litellm import verbose_logger
from litellm._cost import request_model_info
from litellm._internal_context import current_billing_time
from litellm.constants import (
DEFAULT_MAX_LRU_CACHE_SIZE,
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND,
@ -37,17 +39,11 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
_SERVICE_TIER_TO_COST_KEY_SUFFIX,
BilledTokenRates,
CostCalculatorUtils,
calculate_cache_writing_cost,
calculate_cost_component,
generic_cost_per_character,
generic_cost_per_token,
get_batch_cost_rates,
get_billable_input_tokens,
get_regional_uplift_multiplier,
get_service_tier_cost_key,
get_token_type_cost_breakdown,
get_vertex_regional_endpoint_uplift,
parse_prompt_tokens_details,
select_cost_metric_for_model,
)
from litellm.llms.anthropic.cost_calculation import (
@ -239,30 +235,26 @@ def _cost_per_token_custom_pricing_helper(
return None
if custom_cost_per_token is not None:
input_cost_per_token: Final = custom_cost_per_token["input_cost_per_token"]
output_cost_per_token: Final = custom_cost_per_token["output_cost_per_token"]
from litellm._cost import billing_context, price, require_priced
from litellm.rust_bridge import _native
cache_read_input_token_cost: Final = custom_cost_per_token.get(
"cache_read_input_token_cost",
input_cost_per_token,
priced: Final = require_priced(
price(
{"litellm_provider": "custom", **custom_cost_per_token},
_native.UsageInput(
{
"input_cost_per_token": int(max(prompt_tokens - cached_tokens - cache_creation_tokens, 0)),
"cache_read_input_token_cost": int(cached_tokens),
"cache_creation_input_token_cost": int(cache_creation_tokens),
"output_cost_per_token": int(completion_tokens),
},
prompt_tokens=int(prompt_tokens),
),
billing_context(current_billing_time()),
),
"custom pricing",
)
cache_creation_input_token_cost: Final = custom_cost_per_token.get(
"cache_creation_input_token_cost",
input_cost_per_token,
)
regular_prompt_tokens: Final = max(
prompt_tokens - cached_tokens - cache_creation_tokens,
0,
)
input_cost: Final = (
regular_prompt_tokens * input_cost_per_token
+ cached_tokens * cache_read_input_token_cost
+ cache_creation_tokens * cache_creation_input_token_cost
)
output_cost = completion_tokens * output_cost_per_token
return input_cost, output_cost
return priced.input, priced.output
elif custom_cost_per_second is not None:
output_cost = custom_cost_per_second * (response_time_ms or 0.0) / 1000
return 0, output_cost
@ -369,9 +361,9 @@ def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool:
return mode is None or mode in _WALL_CLOCK_PRICED_MODES
def _model_info_for_pricing(model: str, custom_llm_provider: str | None) -> ModelInfoBase | None:
def _model_info_for_pricing(model: str, custom_llm_provider: str | None) -> ModelInfo | None:
try:
return cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
return None
@ -807,18 +799,16 @@ def cost_per_token(
model_info: Final = (
pricing_model_info
if pricing_model_info is not None
else cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
else litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
)
return generic_cost_per_token(
model=model,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,
data_residency=data_residency,
model_info=model_info,
)
if _has_token_or_tiered_pricing(model_info):
return generic_cost_per_token(
model=model,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,
data_residency=data_residency,
)
verbose_logger.debug("No per-token, tiered, or per-second pricing for model=%s; cost is 0", model)
return 0.0, 0.0
def get_replicate_completion_pricing(completion_response: dict, total_time=0.0):
@ -909,7 +899,12 @@ def _bedrock_pricing_info(
)
profile: Final = _cost_map_model_info(invoked_model_id, "bedrock") if invoked_model_id else None
if profile is None or not _declares_rates(profile):
return _cost_map_model_info(registration[0], None) if registration is not None else None
if registration is None:
return None
source: Final = _raw_cost_map_entry(registration[0])
return _cost_map_model_info(
registration[0], "bedrock" if source is not None and "litellm_provider" not in source else None
)
if registration is None or registration[0] == profile["key"]:
return profile
return model_info_with_pricing_overrides(profile, registration[1])
@ -1512,6 +1507,35 @@ def _response_time_ms_for_cost(
return 0.0
def _request_pricing_cost(
*,
model: str,
provider: str | None,
prices: Mapping[str, object] | None,
usage: Usage,
response_time_ms: float | None,
audio_seconds: float,
service_tier: str | None,
data_residency: str | None,
vertex_location: str | None,
) -> tuple[ModelInfo | None, tuple[float, float] | None]:
if prices is None:
return None, None
info: Final = request_model_info(model, provider, prices)
time_cost: Final = _per_second_pricing_cost(model, info, response_time_ms, audio_seconds)
if time_cost is not None:
return info, time_cost
return info, generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider=provider or "custom",
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
model_info=info,
)
def completion_cost(
completion_response: object | None = None,
model: str | None = None,
@ -2032,36 +2056,54 @@ def completion_cost(
if litellm_logging_obj is not None:
request_model_for_cost = litellm_logging_obj.model
(
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
) = cost_per_token(
request_info, request_cost = _request_pricing_cost(
model=model,
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=candidate_pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
provider=custom_llm_provider,
prices=litellm_logging_obj.request_pricing
if custom_pricing and litellm_logging_obj is not None and custom_cost_per_token is None
else None,
usage=cost_per_token_usage_object
or Usage(prompt_tokens=prompt_tokens or 0, completion_tokens=completion_tokens or 0),
response_time_ms=response_time_ms,
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
custom_cost_per_token=custom_cost_per_token,
prompt_characters=prompt_characters,
completion_characters=completion_characters,
cache_creation_input_tokens=cache_creation_input_tokens,
cache_read_input_tokens=cache_read_input_tokens,
usage_object=cost_per_token_usage_object,
call_type=call_type,
audio_transcription_file_duration=audio_transcription_file_duration,
rerank_billed_units=rerank_billed_units,
audio_seconds=audio_transcription_file_duration or 0.0,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
response=completion_response,
custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id),
bedrock_model_info=bedrock_model_info,
)
(
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
) = (
request_cost
if request_cost is not None
else cost_per_token(
model=model,
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=_pricing_provider_for_cost_candidate(
candidate_provider=candidate_pricing_provider,
request_provider=request_pricing_provider,
current_provider=custom_llm_provider,
),
response_time_ms=response_time_ms,
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
custom_cost_per_token=custom_cost_per_token,
prompt_characters=prompt_characters,
completion_characters=completion_characters,
cache_creation_input_tokens=cache_creation_input_tokens,
cache_read_input_tokens=cache_read_input_tokens,
usage_object=cost_per_token_usage_object,
call_type=call_type,
audio_transcription_file_duration=audio_transcription_file_duration,
rerank_billed_units=rerank_billed_units,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
response=completion_response,
custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id),
bedrock_model_info=bedrock_model_info,
)
)
# Get additional costs from provider (e.g., routing fees, infrastructure costs)
@ -2153,7 +2195,7 @@ def completion_cost(
data_residency=data_residency,
vertex_location=vertex_location,
custom_cost_per_token=custom_cost_per_token,
model_info=bedrock_model_info,
model_info=request_info if request_info is not None else bedrock_model_info,
)
_reasoning_cost = _token_type_breakdown.reasoning_cost
_cache_read_cost = _token_type_breakdown.cache_read_cost
@ -2889,32 +2931,26 @@ def default_video_cost_calculator(
return 0.0
def _batch_rate(
model_info: ModelInfo,
key: Literal[
"input_cost_per_audio_token_batches",
"cache_read_input_audio_token_cost_batches",
"input_cost_per_image_token_batches",
"input_cost_per_video_token_batches",
],
fallback: float,
) -> float:
rate: Final = model_info.get(key)
return fallback if rate is None else rate
def _batch_or_half(batch_rate: float | None, standard_rate: float | None, default: float = 0.0) -> float:
if batch_rate is not None:
return batch_rate
if standard_rate is not None:
return standard_rate / 2
return default
def _completion_image_tokens(usage: Usage) -> int:
details: Final = usage.completion_tokens_details
image_tokens: Final = (details.image_tokens or 0) if details is not None else 0
return min(image_tokens, usage.completion_tokens)
def _batch_model_info(model: str, provider: str | None, model_info: ModelInfo | None) -> ModelInfo | None:
if model_info is not None and any(
model_info.get(key) is not None
for key in (
"input_cost_per_token_batches",
"input_cost_per_token",
"output_cost_per_token_batches",
"output_cost_per_token",
)
):
return model_info
try:
global_info: Final = litellm.get_model_info(model=model, custom_llm_provider=provider)
except Exception:
return model_info
if model_info is None:
return global_info
return cast( # cast-ok: both public ModelInfo rows keep their validated field types on merge
ModelInfo, {**global_info, **{key: value for key, value in model_info.items() if value is not None}}
)
def batch_cost_calculator(
@ -2926,143 +2962,28 @@ def batch_cost_calculator(
*,
vertex_location: str | None = None,
) -> tuple[float, float]:
"""
Calculate the cost of a batch job.
from litellm._cost import billing_context, price, require_priced, token_usage
from litellm.litellm_core_utils.llm_cost_calc.utils import uses_inclusive_token_thresholds
Args:
model_info: Optional deployment-level model info containing custom
batch pricing (e.g. input_cost_per_token_batches). When provided,
skips the global litellm.get_model_info() lookup so that
deployment-specific pricing is used.
"""
_, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
verbose_logger.debug(
"Calculating batch cost per token. model=%s, custom_llm_provider=%s",
model,
custom_llm_provider,
)
if model_info is None:
try:
model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception:
model_info = None
elif not any(
model_info.get(k) is not None
for k in (
"input_cost_per_token_batches",
"input_cost_per_token",
"output_cost_per_token_batches",
"output_cost_per_token",
)
):
deployment_info: Final = model_info
try:
global_info: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
if global_info:
model_info = {
**global_info,
**{key: value for key, value in deployment_info.items() if value is not None},
}
except Exception:
pass
if not model_info:
return 0.0, 0.0
batch_rates: Final = get_batch_cost_rates(model_info, usage, custom_llm_provider)
input_cost_per_token: Final = model_info.get("input_cost_per_token")
output_cost_per_token: Final = model_info.get("output_cost_per_token")
total_prompt_cost = 0.0
total_completion_cost = 0.0
if batch_rates.input is not None:
total_prompt_cost = _batch_prompt_cost(
usage,
model_info,
batch_rates.input,
batch_rates.cache_read,
batch_rates.cache_creation,
batch_rates.cache_creation_rates_by_ttl,
)
elif input_cost_per_token:
details: Final = parse_prompt_tokens_details(usage)
cache_read_tokens: Final = details["cache_hit_tokens"]
cache_creation_tokens: Final = details["cache_creation_tokens"]
# Subtract cached tokens from prompt_tokens before calculating cost
# Fixes issue where cached tokens are being charged again
base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens
total_prompt_cost = (
base_input_tokens * (input_cost_per_token) / 2
) # batch cost is usually half of the regular token cost
# Add cache read cost if applicable
cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None)
total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2
cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token
total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token)
image_rate: Final = _batch_or_half(
model_info.get("output_cost_per_image_token_batches"),
model_info.get("output_cost_per_image_token"),
default=text_rate,
)
image_tokens: Final = _completion_image_tokens(usage)
text_tokens: Final = usage.completion_tokens - image_tokens
total_completion_cost = text_tokens * text_rate + image_tokens * image_rate
uplift: Final = get_regional_uplift_multiplier(model_info, data_residency)
if uplift != 1.0:
total_prompt_cost *= uplift
total_completion_cost *= uplift
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift
def _batch_prompt_cost(
usage: Usage,
model_info: ModelInfo,
input_rate: float,
cache_read_rate: float | None,
cache_creation_rate: float | None,
cache_creation_rates_by_ttl: Mapping[str, float],
) -> float:
details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
audio_tokens, image_tokens, video_tokens = (
details["audio_tokens"],
details["image_tokens"],
details["video_tokens"],
)
text_tokens: Final = max(
(usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens - cached_tokens - written_tokens, 0
)
return (
text_tokens * input_rate
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
+ cached_audio_tokens
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
+ (
calculate_cache_writing_cost(
written_tokens,
details["cache_creation_token_details"],
input_rate if cache_creation_rate is None else cache_creation_rate,
cache_creation_rates_by_ttl,
_, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
resolved: Final = _batch_model_info(model, provider, model_info)
row: Final = dict(resolved or {})
pricing_row: Final = row if "litellm_provider" in row else {"litellm_provider": provider or "custom", **row}
priced: Final = require_priced(
price(
pricing_row,
token_usage(usage, pricing_row, inclusive_threshold=uses_inclusive_token_thresholds(provider), batch=True),
billing_context(current_billing_time(), channel="batch"),
factors=(
get_regional_uplift_multiplier(resolved, data_residency),
get_vertex_regional_endpoint_uplift(resolved, vertex_location),
)
if bills_cache_writes
else 0.0
)
if resolved is not None
else (),
),
model,
)
return priced.input, priced.output
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)

View file

@ -626,6 +626,7 @@ class Logging(LiteLLMLoggingBaseClass):
slack_app
custom_pricing: bool = False
custom_cost_per_token: CostPerToken | None = None
request_pricing: Mapping[str, object] | None = None
custom_cost_per_second: float | None = None
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
@ -804,6 +805,11 @@ class Logging(LiteLLMLoggingBaseClass):
self.litellm_params = litellm_params
if kwargs is not None:
from litellm._cost import request_prices
self.request_pricing = request_prices(kwargs)
# Initialize cost breakdown field
self.cost_breakdown: CostBreakdown | None = None
self.billed_token_rates: BilledTokenRates | None = None

File diff suppressed because it is too large Load diff

View file

@ -9,15 +9,12 @@ from pydantic import ValidationError
from litellm.litellm_core_utils.llm_cost_calc.utils import (
generic_cost_per_token,
get_provider_specific_geo_multiplier,
get_provider_specific_speed_multiplier,
get_web_search_requests_from_usage,
)
from litellm.types.llms.base import LiteLLMBaseModel
if TYPE_CHECKING:
from litellm.types.utils import ModelInfo, Usage
import litellm
def cost_per_token(
@ -36,7 +33,7 @@ def cost_per_token(
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
"""
prompt_cost, completion_cost = generic_cost_per_token(
return generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider="anthropic",
@ -44,28 +41,6 @@ def cost_per_token(
model_info=model_info,
)
# Apply provider_specific_entry multipliers for geo/speed routing
try:
effective_info: Final = (
model_info
if model_info is not None
else litellm.get_model_info(model=model, custom_llm_provider="anthropic")
)
geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=effective_info, usage=usage)
speed_multiplier: Final = get_provider_specific_speed_multiplier(model_info=effective_info, usage=usage)
if speed_multiplier != 1.0:
prompt_cost *= speed_multiplier
completion_cost *= speed_multiplier
if geo_multiplier != 1.0:
prompt_cost *= geo_multiplier
completion_cost *= geo_multiplier
except Exception:
pass
return prompt_cost, completion_cost
class _AnthropicServerToolUseProbe(LiteLLMBaseModel):
web_search_requests: int | None = None

View file

@ -5,8 +5,11 @@ Handles Azure AI Foundry Model Router flat cost and other Azure AI specific pric
from typing import Final
from litellm._cost import billing_context, price, require_priced
from litellm._internal_context import current_billing_time
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.rust_bridge import _native
from litellm.types.utils import Usage
from litellm.utils import get_model_info
@ -57,10 +60,14 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl
if not is_azure_model_router(model):
return 0.0
model_info: Final = get_model_info(model=_router_fee_entry_name(model), custom_llm_provider="azure_ai")
router_flat_cost_per_token: Final = model_info.get("input_cost_per_token", 0)
if router_flat_cost_per_token and router_flat_cost_per_token > 0:
return prompt_tokens * router_flat_cost_per_token
return 0.0
return require_priced(
price(
model_info,
_native.UsageInput({"input_cost_per_token": prompt_tokens}, prompt_tokens=prompt_tokens),
billing_context(current_billing_time()),
),
model,
).input
def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float]:

View file

@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Final
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import (
generic_cost_per_token,
get_provider_specific_geo_multiplier,
price_token_usage,
)
if TYPE_CHECKING:
@ -22,12 +22,12 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None)
Follows the same logic as Anthropic's cost per token calculation.
"""
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="bedrock")
prompt_cost, completion_cost = generic_cost_per_token(
return price_token_usage(
model=model,
usage=usage,
custom_llm_provider="bedrock",
service_tier=service_tier,
model_info=model_info,
resolved_model_info=model_info,
current_time=None,
factors=(get_provider_specific_geo_multiplier(model_info=model_info, usage=usage),),
)
geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
return prompt_cost * geo_multiplier, completion_cost * geo_multiplier

View file

@ -1,113 +1,7 @@
"""
Cost calculator for Dashscope Chat models.
Alibaba Model Studio tiered pricing is all-or-nothing: the tier is picked from the
total input tokens of a single request, and every token of that request (input,
cached, cache-creation, output, reasoning) is billed at that one tier's rate.
See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
"""
from collections.abc import Mapping
from dataclasses import dataclass, replace
from datetime import datetime
from typing import Final
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
from litellm.litellm_core_utils.llm_cost_calc.utils import (
TokenRates,
apply_off_peak_pricing,
parse_completion_tokens_details,
parse_prompt_tokens_details,
resolve_explicit_cache_read_cost_rate,
)
from litellm.types.utils import ModelInfo, Usage
from litellm.utils import get_model_info
@dataclass(frozen=True, slots=True)
class TokenBreakdown:
text_tokens: int
cached_tokens: int
cache_creation_tokens: int
completion_tokens: int
reasoning_tokens: int
@property
def total_input_tokens(self) -> int:
return self.text_tokens + self.cached_tokens + self.cache_creation_tokens
def _extract_token_breakdown(usage: Usage) -> TokenBreakdown:
prompt_details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = prompt_details["cache_hit_tokens"]
cache_creation_tokens: Final = prompt_details["cache_creation_tokens"]
text_tokens: Final = max(usage.prompt_tokens - cached_tokens - cache_creation_tokens, 0)
reasoning_tokens: Final = parse_completion_tokens_details(usage)["reasoning_tokens"]
completion_tokens: Final = max((usage.completion_tokens or 0) - reasoning_tokens, 0)
return TokenBreakdown(
text_tokens=text_tokens,
cached_tokens=cached_tokens,
cache_creation_tokens=cache_creation_tokens,
completion_tokens=completion_tokens,
reasoning_tokens=reasoning_tokens,
)
def _flat_rate(value: float | None, fallback_value: float | None) -> float:
if value is None:
return float(fallback_value or 0.0)
return float(value)
def _flat_rates(model_info: ModelInfo) -> TokenRates:
reasoning_rate: Final = model_info.get("output_cost_per_reasoning_token")
return TokenRates(
input_rate=float(model_info.get("input_cost_per_token") or 0.0),
cache_read_rate=_flat_rate(
model_info.get("cache_read_input_token_cost"), model_info.get("input_cost_per_token")
),
cache_creation_rate=_flat_rate(
model_info.get("cache_creation_input_token_cost"), model_info.get("input_cost_per_token")
),
output_rate=float(model_info.get("output_cost_per_token") or 0.0),
reasoning_rate=None if reasoning_rate is None else float(reasoning_rate),
)
def _tier_rates(model_info: ModelInfo, tier: Mapping[str, object]) -> TokenRates:
# A tier that declares output rates keeps the request on them, all-or-nothing. A tier table
# spelling out only input rates would serve every completion for free, so there the model's
# own output rates stand in
flat_rates: Final = _flat_rates(model_info)
tier_declares_output: Final = "output_cost_per_token" in tier
tier_declares_reasoning: Final = "output_cost_per_reasoning_token" in tier
return TokenRates(
input_rate=tier_rate(tier, "input_cost_per_token"),
cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
cache_creation_rate=tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token"),
output_rate=tier_rate(tier, "output_cost_per_token") if tier_declares_output else flat_rates.output_rate,
reasoning_rate=(
tier_rate(tier, "output_cost_per_reasoning_token")
if tier_declares_reasoning
else None
if tier_declares_output
else flat_rates.reasoning_rate
),
)
def _bill(breakdown: TokenBreakdown, rates: TokenRates) -> tuple[float, float]:
prompt_cost: Final = (
(breakdown.text_tokens * rates.input_rate)
+ (breakdown.cached_tokens * rates.cache_read_rate)
+ (breakdown.cache_creation_tokens * rates.cache_creation_rate)
)
completion_cost: Final = (breakdown.completion_tokens * rates.output_rate) + (
breakdown.reasoning_tokens * rates.billed_reasoning_rate
)
return prompt_cost, completion_cost
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import Usage
def cost_per_token(
@ -116,39 +10,9 @@ def cost_per_token(
custom_llm_provider: str = "dashscope",
current_time: datetime | None = None,
) -> tuple[float, float]:
"""
Calculate cost per token for Dashscope models.
Supports both tiered and flat pricing with cached and reasoning tokens, and swaps in the
model's off_peak_pricing rates while one of its windows is open.
Args:
model: Model name without provider prefix
usage: LiteLLM Usage block
custom_llm_provider: The provider id the request resolved to; dashscope or one of its brand aliases
current_time: The moment the request is billed at; defaults to now, UTC
Returns:
Tuple[float, float] - (prompt_cost_in_usd, completion_cost_in_usd)
"""
model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
breakdown: Final = _extract_token_breakdown(usage)
raw_tiers: Final = model_info.get("tiered_pricing")
tiered_pricing: Final = raw_tiers if isinstance(raw_tiers, list) else None
tier: Final = (
select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=breakdown.total_input_tokens)
if tiered_pricing
else None
)
standard_rates: Final = _flat_rates(model_info) if tier is None else _tier_rates(model_info, tier)
rates: Final = apply_off_peak_pricing(model_info, current_time, standard_rates)
explicit_cache_read_rate: Final = resolve_explicit_cache_read_cost_rate(
model_info=model_info,
return generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider=custom_llm_provider,
current_time=current_time,
)
billed_rates: Final = (
rates if explicit_cache_read_rate is None else replace(rates, cache_read_rate=explicit_cache_read_rate)
)
return _bill(breakdown, billed_rates)

View file

@ -1,42 +0,0 @@
from typing import (
Final,
cast, # noqa: TID251 # the derived entry is a dict copy of a ReadOnly TypedDict; no cast-free way to retype it
)
from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
from litellm.types.utils import ModelInfo
def _as_rate(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
return None
try:
return float(value)
except ValueError:
return None
def with_default_cache_read_rate(model_info: ModelInfo) -> ModelInfo:
input_rate: Final = _as_rate(model_info.get("input_cost_per_token"))
if model_info.get("cache_read_input_token_cost") is not None or input_rate is None:
return model_info
cache_read_rate: Final = input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
off_peak: Final = model_info.get("off_peak_pricing")
if off_peak is None or "cache_read_input_token_cost" in off_peak:
return cast(ModelInfo, {**model_info, "cache_read_input_token_cost": cache_read_rate})
off_peak_input_rate: Final = _as_rate(off_peak.get("input_cost_per_token"))
return cast(
ModelInfo,
{
**model_info,
"cache_read_input_token_cost": cache_read_rate,
"off_peak_pricing": {
**off_peak,
"cache_read_input_token_cost": (
off_peak_input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
if off_peak_input_rate is not None
else cache_read_rate
),
},
},
)

View file

@ -6,7 +6,9 @@ Helper util for handling perplexity-specific cost calculation
from datetime import datetime
from typing import Final
from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
from litellm._cost import billing_context, price, require_priced
from litellm._internal_context import current_billing_time
from litellm.rust_bridge import _native
from litellm.types.utils import Usage
from litellm.utils import get_model_info
@ -52,22 +54,31 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
except (ValueError, TypeError):
return default
rates: Final = apply_off_peak_pricing(
model_info,
current_time,
TokenRates(
input_rate=_safe_float_cast(model_info.get("input_cost_per_token")),
output_rate=_safe_float_cast(model_info.get("output_cost_per_token")),
cache_read_rate=0.0,
cache_creation_rate=0.0,
reasoning_rate=None,
),
top_reasoning: Final[object] = getattr(usage, "reasoning_tokens", None)
reasoning_tokens: Final = (
top_reasoning
if isinstance(top_reasoning, int) and top_reasoning > 0
else (usage.completion_tokens_details.reasoning_tokens or 0)
if usage.completion_tokens_details is not None
else 0
)
input_cost_per_token: Final = rates.input_rate
output_cost_per_token: Final = rates.output_rate
## CALCULATE INPUT COST
prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token
priced: Final = require_priced(
price(
{"litellm_provider": "perplexity", **model_info},
_native.UsageInput(
{
"input_cost_per_token": usage.prompt_tokens,
"output_cost_per_token": max(usage.completion_tokens - reasoning_tokens, 0),
"output_cost_per_reasoning_token": reasoning_tokens,
},
prompt_tokens=usage.prompt_tokens,
),
billing_context(current_time if current_time is not None else current_billing_time()),
),
model,
)
prompt_cost: float = priced.input
completion_cost: float = priced.output
## ADD CITATION TOKENS COST (if present)
citation_tokens: Final = getattr(usage, "citation_tokens", 0) or 0
@ -76,24 +87,6 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
citation_cost_per_token: Final = _safe_float_cast(citation_cost_value)
prompt_cost += citation_tokens * citation_cost_per_token
## CALCULATE OUTPUT COST
reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0
if reasoning_tokens == 0 and hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:
reasoning_tokens = getattr(usage.completion_tokens_details, "reasoning_tokens", 0) or 0
reasoning_cost_value: Final = model_info.get("output_cost_per_reasoning_token")
# `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity usage
# convention (codified for the central path in PR #18607). When a reasoning rate is
# configured we subtract before the output-rate multiplication so the reasoning
# tokens are not billed twice.
if reasoning_tokens > 0 and reasoning_cost_value is not None:
non_reasoning_completion_tokens: Final = max(0, (usage.completion_tokens or 0) - reasoning_tokens)
completion_cost: float = non_reasoning_completion_tokens * output_cost_per_token
completion_cost += reasoning_tokens * _safe_float_cast(reasoning_cost_value)
else:
completion_cost = (usage.completion_tokens or 0) * output_cost_per_token
## ADD SEARCH QUERIES COST (if present)
num_search_queries = 0
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:

View file

@ -1,14 +1,18 @@
# What is this?
## Cost calculation for Google AI Studio / Vertex AI models
from collections.abc import Sequence
from typing import Final, Literal
import litellm
from litellm import verbose_logger
from litellm._cost import billing_context, price, require_priced, token_usage
from litellm._internal_context import current_billing_time
from litellm.litellm_core_utils.llm_cost_calc.utils import (
generic_cost_per_token,
get_vertex_regional_endpoint_uplift,
is_above_128k,
)
from litellm.rust_bridge import _native
from litellm.types.utils import ModelInfo, Usage
"""
@ -58,6 +62,30 @@ def cost_router(
return "cost_per_character"
def _token_component_cost(
model: str,
model_info: ModelInfo,
usage: Usage,
direction: Literal["input", "output"],
service_tier: str | None,
) -> float:
if (
model_info.get("input_cost_per_token_above_128k_tokens") is not None
or model_info.get("output_cost_per_token_above_128k_tokens") is not None
):
input_cost, output_cost = _handle_128k_pricing(model_info, usage, direction=direction)
return input_cost if direction == "input" else output_cost
priced: Final = require_priced(
price(
model_info,
token_usage(usage, model_info, direction=direction),
billing_context(current_billing_time(), service_tier),
),
model,
)
return priced.input if direction == "input" else priced.output
def cost_per_character(
model: str,
custom_llm_provider: str,
@ -91,12 +119,7 @@ def cost_per_character(
## CALCULATE INPUT COST
if prompt_characters is None:
prompt_cost, _ = cost_per_token(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
prompt_cost = _token_component_cost(model, model_info, usage, "input", service_tier)
else:
try:
if (
@ -123,21 +146,11 @@ def cost_per_character(
"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - %s\nDefaulting to None",
e,
)
prompt_cost, _ = cost_per_token(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
prompt_cost = _token_component_cost(model, model_info, usage, "input", service_tier)
## CALCULATE OUTPUT COST
if completion_characters is None:
_, completion_cost = cost_per_token(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
completion_cost = _token_component_cost(model, model_info, usage, "output", service_tier)
else:
completion_tokens: Final = usage.completion_tokens
try:
@ -164,12 +177,7 @@ def cost_per_character(
"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - %s\nDefaulting to None",
e,
)
_, completion_cost = cost_per_token(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
service_tier=service_tier,
)
completion_cost = _token_component_cost(model, model_info, usage, "output", service_tier)
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
return prompt_cost * vertex_uplift, completion_cost * vertex_uplift
@ -178,27 +186,27 @@ def cost_per_character(
def _handle_128k_pricing(
model_info: ModelInfo,
usage: Usage,
*,
factors: Sequence[float] = (),
direction: Literal["input", "output"] | None = None,
) -> tuple[float, float]:
## CALCULATE INPUT COST
input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens")
output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens")
prompt_tokens: Final = usage.prompt_tokens
completion_tokens: Final = usage.completion_tokens
if is_above_128k(tokens=prompt_tokens) and input_cost_per_token_above_128k_tokens is not None:
prompt_cost = prompt_tokens * input_cost_per_token_above_128k_tokens
else:
prompt_cost = prompt_tokens * (model_info["input_cost_per_token"] or 0.0)
## CALCULATE OUTPUT COST
output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens")
if is_above_128k(tokens=completion_tokens) and output_cost_per_token_above_128k_tokens is not None:
completion_cost = completion_tokens * output_cost_per_token_above_128k_tokens
else:
completion_cost = completion_tokens * (model_info["output_cost_per_token"] or 0.0)
return prompt_cost, completion_cost
result: Final = require_priced(
price(
model_info,
_native.UsageInput(
{
**({"input_cost_per_token": usage.prompt_tokens} if direction != "output" else {}),
**({"output_cost_per_token": usage.completion_tokens} if direction != "input" else {}),
},
prompt_tokens=usage.prompt_tokens,
),
billing_context(current_billing_time()),
factors=factors,
directional_token_threshold=128000,
),
model_info.get("key", "vertex_ai"),
)
return result.input, result.output
def cost_per_token(
@ -235,12 +243,11 @@ def cost_per_token(
input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens")
output_cost_per_token_above_128k_tokens: Final = model_info.get("output_cost_per_token_above_128k_tokens")
if input_cost_per_token_above_128k_tokens is not None or output_cost_per_token_above_128k_tokens is not None:
prompt_cost_128k, completion_cost_128k = _handle_128k_pricing(
return _handle_128k_pricing(
model_info=model_info,
usage=usage,
factors=(get_vertex_regional_endpoint_uplift(model_info, vertex_location),),
)
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
return prompt_cost_128k * vertex_uplift, completion_cost_128k * vertex_uplift
return generic_cost_per_token(
model=model,

View file

@ -155,7 +155,6 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
BaseUrlRoutingHint,
CustomPricingLiteLLMParams,
ModelResponseStream,
ProviderRoutingHints,
RawRequestTypedDict,
@ -1115,7 +1114,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o
)
from litellm.responses.bridge_check import (
bridge_reasoning_effort,
keep_registered_mode,
str_keyed_mapping,
)
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
@ -1232,93 +1230,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]:
return values
def _build_custom_pricing_entry(
custom_llm_provider: str,
kwargs: dict,
model_info: dict | None = None,
) -> dict:
"""Build a complete model cost entry from kwargs and model_info.
Collects all CustomPricingLiteLLMParams fields present in kwargs and
merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
so that register_model() receives the full pricing configuration.
"""
entry: Final[dict] = {"litellm_provider": custom_llm_provider}
for field_name in CustomPricingLiteLLMParams.model_fields:
value = kwargs.get(field_name)
if value is not None:
entry[field_name] = value
if model_info and isinstance(model_info, dict):
for key in ("mode", "supports_prompt_caching", "max_tokens"):
if key in model_info and model_info[key] is not None:
entry.setdefault(key, model_info[key])
return entry
def _get_router_deployment_id(kwargs: dict) -> str | None:
for metadata_key in ("litellm_metadata", "metadata"):
metadata = kwargs.get(metadata_key) or {}
if not isinstance(metadata, dict):
continue
deployment_model_info = metadata.get("model_info") or {}
if not isinstance(deployment_model_info, dict):
continue
deployment_id = deployment_model_info.get("id")
if deployment_id is not None:
return str(deployment_id)
return None
def _register_custom_pricing_for_request(
model: str,
custom_llm_provider: str,
kwargs: dict,
model_info: dict | None,
) -> None:
"""Register per-request custom pricing in litellm.model_cost.
Router-originated requests (identified by the deployment id the router puts
in metadata) get their full pricing registered under that unique id only;
the shared ``{provider}/{model}`` key receives the entry with pricing fields
stripped, mirroring Router._create_deployment. This keeps one deployment's
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
pricing used by sibling deployments of the same backend model. Direct SDK
calls keep the legacy behavior of registering the shared key with pricing.
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
deployment registration, since a call routes by its own ``model_info``.
"""
entry: Final = _build_custom_pricing_entry(
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
shared_key: Final = f"{custom_llm_provider}/{model}"
deployment_id: Final = _get_router_deployment_id(kwargs)
if deployment_id is None:
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
return
shared_entry: Final = keep_registered_mode(
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
)
litellm.register_model(
{deployment_id: entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
add_to_provider_model_sets=False,
)
litellm.register_model(
{shared_key: shared_entry},
persist_across_reloads=False,
warning_display_name=shared_key,
custom_llm_provider=custom_llm_provider,
add_to_provider_model_sets=False,
)
def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None:
return ctx.metadata
@ -5750,19 +5661,6 @@ def completion(
"aws_region_name", None
) # support region-based pricing for bedrock
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if (
(input_cost_per_token is not None and output_cost_per_token is not None)
or input_cost_per_second is not None
or output_cost_per_second is not None
or cost_per_second is not None
):
_register_custom_pricing_for_request(
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=model_info,
)
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
custom_prompt_dict = {}
if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token:
@ -6705,12 +6603,6 @@ def embedding(
headers.update(proxy_headers)
except Exception as e:
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
### CUSTOM MODEL COST ###
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
cost_per_second: Final = kwargs.get("cost_per_second", None)
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
openai_params: Final = [
"user",
"dimensions",
@ -6757,20 +6649,6 @@ def embedding(
**non_default_params,
)
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
if (
(input_cost_per_token is not None and output_cost_per_token is not None)
or input_cost_per_second is not None
or output_cost_per_second is not None
or cost_per_second is not None
):
_register_custom_pricing_for_request(
model=model,
custom_llm_provider=custom_llm_provider,
kwargs=kwargs,
model_info=kwargs.get("model_info"),
)
logging: Final[LiteLLMLoggingObj] = litellm_logging_obj
logging.update_environment_variables(
model=model,

View file

@ -26,6 +26,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import NotRequired, ReadOnly, Required, TypedDict, Unpack
import litellm
from litellm._catalog import is_priced
from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.caching.dual_cache import DualCache, LimitedSizeOrderedDict
@ -605,7 +606,7 @@ def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None)
# See: https://github.com/BerriAI/litellm/issues/24770
safe_name = str(model_name).replace("\n", "").replace("\r", "")
served_deployments = _deployments_served_for(model_name, llm_router)
if not _is_cost_explicitly_configured(served_deployments):
if not (deployment_prices := tuple(_priced_deployment_infos(served_deployments))):
verbose_proxy_logger.debug(
"Model %s has zero cost but no explicit cost "
"configuration in model_cost entry — treating as unknown "
@ -616,7 +617,11 @@ def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None)
zero_cost_cache[model_name] = False
return False
if _has_explicit_positive_cost(served_deployments):
if any(
_is_positive_cost(info.get("input_cost_per_token"))
or _is_positive_cost(info.get("output_cost_per_token"))
for info in deployment_prices
):
verbose_proxy_logger.debug(
"Model %s routes to a deployment with an explicit positive per-token price (enforce budget)",
safe_name,
@ -696,47 +701,12 @@ def _has_ptu_flat_cost(deployments: Sequence[DeploymentTypedDict]) -> bool:
return False
def _is_cost_explicitly_configured(deployments: Sequence[DeploymentTypedDict]) -> bool:
"""
Check if any of the deployments has cost fields explicitly set in its
litellm.model_cost entry.
When Router._create_deployment() registers a model not in the global
cost map, it creates a sparse entry like {"id": "<hash>"} with no cost
fields. _get_model_info_helper() then defaults missing costs to 0.
This function detects that scenario by checking the raw model_cost entry.
It also reaches a deployment that prices itself through its ``model_info`` block, whose entry
lands in the cost map under the deployment id.
"""
def _priced_deployment_infos(deployments: Sequence[DeploymentTypedDict]) -> Iterator[Mapping[str, object]]:
for deployment in deployments:
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
if model_id is None:
continue
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry:
return True
return False
def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> bool:
"""Whether any of the deployments carries an explicit positive per-token price.
The group's price is read from the deployments named after the routed group and from
that group's own alias target, while an alias chain is served from the deployments named
after its routed group or from a wildcard route; a priced route among those serves the
request at its own rate whatever the group's price reads.
"""
for deployment in deployments:
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
if model_id is None:
continue
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
if _is_positive_cost(raw_entry.get("input_cost_per_token")) or _is_positive_cost(
raw_entry.get("output_cost_per_token")
if isinstance(model_id := (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id"), str) and is_priced(
model_id
):
return True
return False
yield litellm.get_model_info(model=model_id)
_EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({})
@ -798,7 +768,7 @@ def model_has_no_cost_mapping(model: str | None, llm_router: Router | None) -> b
if _model_group_has_pricing(model=model, llm_router=llm_router):
return False
return not _is_cost_explicitly_configured(llm_router.get_model_list(model_name=model) or ())
return not any(_priced_deployment_infos(llm_router.get_model_list(model_name=model) or ()))
def _unpriced_models_in_request(model: str | list[str] | None, llm_router: Router | None) -> tuple[str, ...]:

View file

@ -203,7 +203,7 @@ from litellm.types.utils import (
is_server_derived_pricing_key,
pricing_override_fields,
)
from litellm.utils import cost_map_omits_token_price, load_credentials_from_list, with_named_credential
from litellm.utils import load_credentials_from_list, with_named_credential
if TYPE_CHECKING:
from aiohttp import ClientSession
@ -15996,7 +15996,11 @@ def _enrich_model_info_with_litellm_data(
discovered_model_info: Final = (
llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({})
)
unpriced: Final = cost_map_omits_token_price(model_info.get("id"), litellm_model_info.get("key"))
from litellm._catalog import is_priced
unpriced: Final = not any(
is_priced(key) for key in (model_info.get("id"), litellm_model_info.get("key")) if isinstance(key, str)
)
stamped_model_info: Final = MappingProxyType(
{**model_info, **_pricing_override_stamps(model_info, model.get("litellm_params") or MappingProxyType({}))}
)

View file

@ -52,6 +52,7 @@ from typing_extensions import assert_never, overload
import litellm
import litellm.litellm_core_utils.exception_mapping_utils
from litellm import get_secret_str
from litellm._catalog import is_priced
from litellm._internal_context import service_target, with_service_target
from litellm._logging import verbose_router_logger
from litellm._uuid import uuid
@ -11548,7 +11549,7 @@ class Router:
A strategy-router alias is never the deployment actually called or
billed, so custom pricing configured on it must not become a cost-map
price: an explicit zero would let ``_is_cost_explicitly_configured``
price: an explicit zero would let the free-model budget policy
treat the alias as a genuinely free model and waive budget checks for
requests that route to (and bill as) a real deployment.
"""
@ -12321,7 +12322,7 @@ class Router:
2. If not, check if litellm model name is in model info
3. If not, return None
"""
from litellm.utils import cost_map_omits_token_price, update_dictionary
from litellm.utils import update_dictionary
model_info: ModelInfo | None = None
custom_model_info: dict | None = None
@ -12386,7 +12387,7 @@ class Router:
builtin_key: Final = (
litellm_model_name_model_info.get("key") if litellm_model_name_model_info is not None else None
)
if cost_map_omits_token_price(model_id, builtin_key, base_model_key):
if not any(is_priced(key) for key in (model_id, builtin_key, base_model_key) if isinstance(key, str)):
return cast( # cast-ok: TypedDict spread with overridden keys loses its type
ModelInfo, {**model_info, "input_cost_per_token": None, "output_cost_per_token": None}
)

View file

@ -3660,22 +3660,6 @@ def set_live_deployment_replay(replay: Callable[[], None]) -> None:
_LiveDeploymentReplay.callbacks = (replay,)
def cost_map_omits_token_price(*keys: object) -> bool:
"""Whether the raw ``litellm.model_cost`` entries under ``keys`` exist but none carries a per-token price.
``get_model_info`` substitutes 0 for a missing price, which reads exactly like a declared
zero. Surfaces that report pricing use this to keep an unpriced deployment at ``None``.
"""
entries: Final = tuple(
entry
for entry in (litellm.model_cost.get(key) for key in keys if isinstance(key, str))
if isinstance(entry, dict)
)
return len(entries) > 0 and not any(
"input_cost_per_token" in entry or "output_cost_per_token" in entry for entry in entries
)
def register_model(
model_cost: str | dict,
*,
@ -6322,25 +6306,8 @@ def get_model_info_helper(
if _model_info is None or key is None:
raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider))
_input_cost_per_token: float | None = _model_info.get("input_cost_per_token")
if _input_cost_per_token is None:
# default value to 0, be noisy about this
verbose_logger.debug(
"model=%s, custom_llm_provider=%s has no input_cost_per_token in model_cost_map. Defaulting to 0.",
model,
custom_llm_provider,
)
_input_cost_per_token = 0
_output_cost_per_token: float | None = _model_info.get("output_cost_per_token")
if _output_cost_per_token is None:
# default value to 0, be noisy about this
verbose_logger.debug(
"model=%s, custom_llm_provider=%s has no output_cost_per_token in model_cost_map. Defaulting to 0.",
model,
custom_llm_provider,
)
_output_cost_per_token = 0
_input_cost_per_token: Final = _model_info.get("input_cost_per_token")
_output_cost_per_token: Final = _model_info.get("output_cost_per_token")
returned_model_info: Final = ModelInfoBase(
key=key,

View file

@ -13,6 +13,90 @@ from litellm.rust_bridge._native import ModelNotFound, ResolvedModel
from litellm.utils import ModelNotMappedError, _invalidate_model_cost_lowercase_map, get_model_cost_key
@pytest.mark.parametrize(
("prices", "expected"),
(
({}, False),
({"input_cost_per_token": None, "output_cost_per_token": None}, False),
({"input_cost_per_token": 0, "output_cost_per_token": 0}, True),
({"input_cost_per_token": 0.25}, True),
({"output_cost_per_token": 0.5}, True),
({"input_cost_per_token": -1}, False),
({"input_cost_per_token": "invalid-price"}, False),
({"input_cost_per_second": 1.0}, False),
({"cache_read_input_token_cost": 0.0}, True),
({"cache_creation_input_token_cost_above_1hr": 0.0}, True),
({"output_cost_per_reasoning_token": 0.5}, True),
({"input_cost_per_audio_token": 0.5}, True),
({"output_cost_per_image_token": 0.5}, True),
({"input_cost_per_token_priority": 0.5}, True),
({"input_cost_per_token_batches": 0.5}, True),
({"input_cost_per_token_above_128k_tokens": 0.5}, True),
({"tiered_pricing": [{"range": [0, 200], "input_cost_per_token": 0.5}]}, True),
({"tiered_pricing": [{"range": [100, 200], "input_cost_per_token": 0.5}]}, False),
({"off_peak_pricing": {"hours_utc": "01:00-02:00", "input_cost_per_token": 0.5}}, True),
(
{
"input_cost_per_token": 1.0,
"tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 2.0}],
},
True,
),
),
)
def test_is_priced_distinguishes_explicit_token_rates(
monkeypatch: pytest.MonkeyPatch, prices: dict[str, object], expected: bool
) -> None:
from litellm._catalog import is_priced
monkeypatch.setattr(litellm, "model_cost", {"openai/price-probe": {"litellm_provider": "openai", **prices}})
assert is_priced("price-probe", "openai") is expected
assert is_priced("unmapped-price-probe", "openai") is False
def test_is_priced_tracks_selected_live_row_and_tombstones(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm._catalog import is_priced
rows: Final[dict[str, dict[str, object]]] = {
"price-probe": {"litellm_provider": "openai", "input_cost_per_token": 1},
"openai/price-probe": {"litellm_provider": "openai"},
}
monkeypatch.setattr(litellm, "model_cost", rows)
assert is_priced("price-probe", "openai") is False
rows["openai/price-probe"]["input_cost_per_token"] = 0
assert is_priced("price-probe", "openai") is True
rows["openai/price-probe"] = {"litellm_provider": "openai"}
assert is_priced("price-probe", "openai") is False
del rows["openai/price-probe"]
assert is_priced("price-probe", "openai") is True
del rows["price-probe"]
assert is_priced("price-probe", "openai") is False
def test_providerless_deployment_prices_without_writing_provider_context(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm._catalog import is_priced
row: Final = {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}
monkeypatch.setitem(litellm.model_cost, "providerless-priced-deployment", row)
assert is_priced("providerless-priced-deployment", "openai") is True
assert litellm.cost_per_token(
"providerless-priced-deployment", 10, 20, custom_llm_provider="openai"
) == pytest.approx((10.0, 40.0))
assert row == {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}
@pytest.mark.parametrize("provider", (None, "", 123))
def test_presence_does_not_repair_an_explicit_invalid_provider(
monkeypatch: pytest.MonkeyPatch, provider: object
) -> None:
from litellm._catalog import is_priced
row: Final = {"litellm_provider": provider, "input_cost_per_token": 1.0, "output_cost_per_token": 2.0}
monkeypatch.setitem(litellm.model_cost, "openai/invalid-provider-price-probe", row)
assert is_priced("invalid-provider-price-probe", "openai") is False
assert row == {"litellm_provider": provider, "input_cost_per_token": 1.0, "output_cost_per_token": 2.0}
def test_model_info_preserves_every_bundled_row() -> None:
for key, row in litellm.model_cost.items():
result: Final = _read.resolve(key)

View file

@ -0,0 +1,309 @@
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Final
import pytest
import litellm
from litellm._cost import billing_context, evaluate, price, require_priced, token_usage
from litellm._internal_context import pinned_billing_time
from litellm.exceptions import ModelNotMappedError
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.rust_bridge import _native
from litellm.types.utils import ModelResponse, Usage
@pytest.mark.parametrize(
("tier_prices", "expected"),
(
({"output_cost_per_token": 4.0}, (10.0, 20.0)),
({"input_cost_per_token": None, "output_cost_per_token": 4.0}, None),
({"input_cost_per_token": 0.0, "output_cost_per_token": 4.0}, (0.0, 40.0)),
({"input_cost_per_token": 3.0}, (30.0, 20.0)),
({"input_cost_per_token": 3.0, "output_cost_per_token": None}, None),
({"input_cost_per_token": 3.0, "output_cost_per_token": 0.0}, (30.0, 0.0)),
),
ids=("input-absent", "input-null", "input-zero", "output-absent", "output-null", "output-zero"),
)
def test_tier_declared_null_is_unpriced_and_explicit_zero_remains_free(
monkeypatch: pytest.MonkeyPatch, tier_prices: Mapping[str, object], expected: tuple[float, float] | None
) -> None:
model: Final = "openai/m5-tier-presence"
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
"tiered_pricing": [{"range": [0, 100], **tier_prices}],
},
)
response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=10))
if expected is None:
with pytest.raises(ModelNotMappedError):
litellm.cost_per_token(model, 10, 10, custom_llm_provider="openai")
with pytest.raises(ModelNotMappedError):
litellm.completion_cost(response, model=model, custom_llm_provider="openai")
else:
assert litellm.cost_per_token(model, 10, 10, custom_llm_provider="openai") == pytest.approx(expected)
assert litellm.completion_cost(response, model=model, custom_llm_provider="openai") == pytest.approx(
sum(expected)
)
@pytest.mark.parametrize(
("rates", "usage", "expected"),
(
(
{
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
"tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 3.0, "output_cost_per_token": 4.0}],
},
Usage(prompt_tokens=10, completion_tokens=10),
(30.0, 40.0),
),
(
{
"cache_read_input_token_cost": 9.0,
"tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 1.0, "output_cost_per_token": 2.0}],
},
Usage(prompt_tokens=100, completion_tokens=10, prompt_tokens_details={"cached_tokens": 40}),
(100.0, 20.0),
),
(
{
"cache_creation_input_token_cost_above_1hr": 9.0,
"tiered_pricing": [
{
"range": [0, 100],
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
"cache_creation_input_token_cost": 3.0,
}
],
},
Usage(
prompt_tokens=100,
completion_tokens=10,
prompt_tokens_details={
"cache_creation_tokens": 20,
"cache_creation_token_details": {"ephemeral_5m_input_tokens": 10, "ephemeral_1h_input_tokens": 10},
},
),
(140.0, 20.0),
),
),
ids=("flat-and-tier", "tier-cache-read-fallback", "tier-hourly-cache-write-fallback"),
)
def test_public_token_pricing_preserves_tier_local_fallbacks(
monkeypatch: pytest.MonkeyPatch, rates: Mapping[str, object], usage: Usage, expected: tuple[float, float]
) -> None:
model: Final = "openai/m5-tier-fallback"
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **rates})
assert litellm.cost_per_token(
model, usage.prompt_tokens, usage.completion_tokens, custom_llm_provider="openai", usage_object=usage
) == pytest.approx(expected)
assert litellm.completion_cost(
ModelResponse(model=model, usage=usage), model=model, custom_llm_provider="openai"
) == pytest.approx(sum(expected))
@pytest.mark.parametrize("stream", (False, True))
@pytest.mark.parametrize(
("kind", "rates", "expected"),
(
("unknown", None, None),
("metadata", {}, None),
("free", {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, 0.0),
("paid", {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}, 50.0),
),
)
def test_unknown_free_and_priced_public_surfaces(
monkeypatch: pytest.MonkeyPatch,
stream: bool,
kind: str,
rates: Mapping[str, float] | None,
expected: float | None,
) -> None:
model: Final = f"openai/m5-{kind}"
if rates is not None:
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **rates})
info: Final = litellm.get_model_info(model, custom_llm_provider="openai")
assert info["input_cost_per_token"] == rates.get("input_cost_per_token")
assert info["output_cost_per_token"] == rates.get("output_cost_per_token")
usage: Final = Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30)
response: Final = ModelResponse(model=model, usage=usage)
logging: Final = Logging(
model=model,
messages=[{"role": "user", "content": "hello"}],
stream=stream,
call_type="completion",
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
litellm_call_id="m5-price-surfaces",
function_id="m5-price-surfaces",
)
logging.update_environment_variables(litellm_params={}, optional_params={}, custom_llm_provider="openai")
if expected is None:
with pytest.raises(ModelNotMappedError):
litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai", usage_object=usage)
with pytest.raises(ModelNotMappedError):
litellm.completion_cost(response, model=model, custom_llm_provider="openai")
assert logging.response_cost_calculator(response) is None
assert logging.model_call_details.get("response_cost_failure_debug_information") is not None
else:
split: Final = litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai", usage_object=usage)
assert split == pytest.approx((expected / 5, expected * 4 / 5))
assert litellm.completion_cost(response, model=model, custom_llm_provider="openai") == pytest.approx(expected)
assert logging.response_cost_calculator(response) == pytest.approx(expected)
def test_public_pricing_reads_direct_row_updates(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "openai/m5-live-update"
row: Final = {"litellm_provider": "openai", "input_cost_per_token": 1.0, "output_cost_per_token": 2.0}
monkeypatch.setitem(litellm.model_cost, model, row)
assert litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai") == pytest.approx((10.0, 40.0))
monkeypatch.setitem(row, "input_cost_per_token", 3.0)
assert litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai") == pytest.approx((30.0, 40.0))
def test_invalid_price_is_not_authorized_by_permissive_model_info(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "openai/m5-invalid-row"
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": "openai",
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
"input_cost_per_audio_token": "invalid",
},
)
assert litellm.get_model_info(model, custom_llm_provider="openai")["input_cost_per_token"] == 1.0
with pytest.raises(ModelNotMappedError, match="input_cost_per_audio_token"):
litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai")
def test_custom_cost_uses_ephemeral_row_and_native_cache_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "openai/m5-custom-cost"
row: Final = {"litellm_provider": "openai", "input_cost_per_token": 10.0, "output_cost_per_token": 20.0}
monkeypatch.setitem(litellm.model_cost, model, row)
usage: Final = Usage(
prompt_tokens=100,
completion_tokens=10,
prompt_tokens_details={"cached_tokens": 40, "cache_creation_tokens": 20},
)
assert litellm.cost_per_token(
model,
100,
10,
usage_object=usage,
custom_cost_per_token={"input_cost_per_token": 1.0, "output_cost_per_token": 2.0},
) == pytest.approx((100.0, 20.0))
assert litellm.model_cost[model] is row
assert row["input_cost_per_token"] == 10.0
assert litellm.cost_per_token(model, 100, 10, custom_llm_provider="openai") == pytest.approx((1000.0, 200.0))
@pytest.mark.parametrize("tier", ("priority", "fast", " FAST "))
def test_billing_context_normalizes_priority_alias(tier: str) -> None:
row: Final = {"litellm_provider": "openai", "input_cost_per_token": 1.0, "input_cost_per_token_priority": 3.0}
usage: Final = Usage(prompt_tokens=10, completion_tokens=0)
result: Final = require_priced(
price(row, token_usage(usage, row), billing_context(datetime(2026, 1, 1), tier)),
"priority",
)
assert (result.input, result.output) == (30.0, 0.0)
def test_adapter_preserves_missing_meter_and_native_fallback_result() -> None:
when: Final = billing_context(datetime(2026, 1, 1))
missing: Final = price({"litellm_provider": "openai"}, _native.UsageInput({"input_cost_per_token": 1}), when)
assert isinstance(missing, _native.Unpriced)
assert tuple(missing.missing) == ("input_cost_per_token",)
fallback: Final = require_priced(
price(
{"litellm_provider": "openai", "input_cost_per_token": 2.0},
_native.UsageInput({"cache_read_input_token_cost": 10}),
when,
),
"cache fallback",
)
assert fallback.input == 20.0
assert fallback.lines[0].fallback_from is not None
@pytest.mark.parametrize("factor", (1.0, 1.1))
@pytest.mark.parametrize(
("prompt", "completion", "cache", "tier", "expected", "rates"),
(
(10, 10, 0, None, (10, 20), (1, 2)),
(128000, 128000, 0, None, (128000, 256000), (1, 2)),
(128001, 10, 0, None, (384003, 20), (3, 2)),
(10, 128001, 0, None, (10, 512004), (1, 4)),
(128001, 128001, 0, None, (384003, 512004), (3, 4)),
(100, 10, 80, None, (100, 20), (1, 2)),
(100, 10, 0, "priority", (100, 20), (1, 2)),
),
)
def test_vertex_directional_route_uses_real_aggregate_quantities_and_native_projected_rates(
monkeypatch: pytest.MonkeyPatch,
factor: float,
prompt: int,
completion: int,
cache: int,
tier: str | None,
expected: tuple[float, float],
rates: tuple[float, float],
) -> None:
model: Final = "vertex_ai/gemini-2-directional-test"
row: Final = {
"litellm_provider": "vertex_ai",
"mode": "chat",
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
"input_cost_per_token_above_128k_tokens": 3.0,
"output_cost_per_token_above_128k_tokens": 4.0,
"cache_read_input_token_cost": 0.1,
"input_cost_per_token_priority": 5.0,
"output_cost_per_token_priority": 6.0,
"regional_endpoint_uplift_multiplier": factor,
"off_peak_pricing": {"hours_utc": "00:00-06:00", "input_cost_per_token": 0.1, "output_cost_per_token": 0.2},
}
monkeypatch.setitem(litellm.model_cost, model, row)
shaped: Final = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai")
usage: Final = Usage(
prompt_tokens=prompt, completion_tokens=completion, prompt_tokens_details={"cached_tokens": cache}
)
when: Final = datetime(2026, 1, 1, 1, tzinfo=timezone.utc)
with pinned_billing_time(when):
actual: Final = litellm.cost_per_token(
model,
prompt,
completion,
custom_llm_provider="vertex_ai",
usage_object=usage,
service_tier=tier,
vertex_location="us-east5",
)
assert actual == pytest.approx((expected[0] * factor, expected[1] * factor))
for source in (row, shaped):
evaluated: Final = evaluate(
source,
_native.UsageInput(
{"input_cost_per_token": prompt, "output_cost_per_token": completion}, prompt_tokens=prompt
),
billing_context(when, tier),
project=("input_cost_per_token", "output_cost_per_token"),
factors=(factor,),
directional_token_threshold=128000,
)
assert isinstance(evaluated, _native.PriceEvaluation)
priced: Final = require_priced(evaluated.pricing, model)
assert (priced.input, priced.output) == pytest.approx(actual)
assert tuple(selected.unit_price for selected in evaluated.rates) == pytest.approx(
(rates[0] * factor, rates[1] * factor)
)

View file

@ -11,13 +11,14 @@ Tests the cost calculation for Dashscope models including:
import math
import os
from datetime import datetime, timezone
from typing import Final
import pytest
# Add the project root to Python path
import litellm
from litellm.exceptions import ModelNotMappedError
from litellm.litellm_core_utils.llm_cost_calc.utils import (
resolve_explicit_cache_read_cost_rate,
calculate_prompt_caching_savings,
generic_cost_per_token,
get_token_type_cost_breakdown,
@ -43,32 +44,35 @@ class TestDashscopeCostCalculator:
litellm.model_cost = litellm.get_model_cost_map(url="")
@pytest.mark.parametrize(
"prompt_tokens_details",
("prompt_tokens_details", "expected"),
[
{"cached_tokens": 0, "cache_type": "ephemeral"},
{"cached_tokens": 600},
{"cached_tokens": 600, "cache_type": "unknown-future-type"},
({"cached_tokens": 0, "cache_type": "ephemeral"}, 0.001),
({"cached_tokens": 600}, 0.00052),
({"cached_tokens": 600, "cache_type": "unknown-future-type"}, 0.00052),
({"cached_tokens": 600, "cache_type": "ephemeral"}, 0.00046),
],
)
def test_explicit_cache_rate_requires_an_ephemeral_cache_hit(
self,
prompt_tokens_details: dict[str, int | str],
):
assert (
resolve_explicit_cache_read_cost_rate(
model_info={
"key": "qwen-cache-rate-guard-test",
"explicit_cache_read_input_token_cost": 1e-07,
},
usage=Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details=prompt_tokens_details,
),
current_time=None,
)
is None
)
expected: float,
) -> None:
assert generic_cost_per_token(
model="qwen-cache-rate-guard-test",
custom_llm_provider="dashscope",
model_info={
"key": "qwen-cache-rate-guard-test",
"input_cost_per_token": 1e-06,
"cache_read_input_token_cost": 2e-07,
"explicit_cache_read_input_token_cost": 1e-07,
},
usage=Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details=prompt_tokens_details,
),
current_time=None,
) == pytest.approx((expected, 0))
@pytest.mark.parametrize(
("model_info", "current_time", "expected_cache_rate"),
@ -100,7 +104,7 @@ class TestDashscopeCostCalculator:
],
},
None,
None,
2e-07,
),
(
{
@ -109,7 +113,7 @@ class TestDashscopeCostCalculator:
"cache_read_input_token_cost": 2e-07,
},
None,
None,
2e-07,
),
(
{
@ -179,7 +183,7 @@ class TestDashscopeCostCalculator:
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
8e-08,
None,
),
(
{
@ -191,34 +195,49 @@ class TestDashscopeCostCalculator:
},
},
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
1e-07,
None,
),
],
)
def test_provider_neutral_cache_rate_resolver_handles_qwen_rate_fallbacks(
def test_qwen_cache_mode_uses_native_fallbacks_and_rejects_invalid_rows(
self,
model_info: ModelInfo,
current_time: datetime | None,
expected_cache_rate: float | None,
):
usage = Usage(
) -> None:
usage: Final = Usage(
prompt_tokens=1000,
completion_tokens=0,
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
)
cache_rate = resolve_explicit_cache_read_cost_rate(
if expected_cache_rate is None:
with pytest.raises(ModelNotMappedError):
generic_cost_per_token(
model="qwen-cache-rates",
custom_llm_provider="dashscope",
model_info=model_info,
usage=usage,
current_time=current_time,
)
return
cost: Final = generic_cost_per_token(
model="qwen-cache-rates",
custom_llm_provider="dashscope",
model_info=model_info,
usage=usage,
current_time=current_time,
)
assert cache_rate == expected_cache_rate
assert cost == pytest.approx((400e-06 + 600 * expected_cache_rate, 0))
def test_provider_neutral_cache_rate_resolver_handles_service_tier_rate(self):
cache_rate = resolve_explicit_cache_read_cost_rate(
def test_ephemeral_cache_uses_the_service_tier_rate(self) -> None:
cost: Final = generic_cost_per_token(
model="qwen-service-tier-mode-rate-test",
custom_llm_provider="dashscope",
model_info={
"key": "qwen-service-tier-mode-rate-test",
"input_cost_per_token": 1e-06,
"explicit_cache_read_input_token_cost_priority": 3e-07,
},
usage=Usage(
@ -230,7 +249,7 @@ class TestDashscopeCostCalculator:
service_tier="priority",
)
assert cache_rate == 3e-07
assert cost == pytest.approx((0.00058, 0))
def test_dashscope_flat_pricing_fallback(self):
"""
@ -239,9 +258,7 @@ class TestDashscopeCostCalculator:
"""
usage = Usage(prompt_tokens=1000, completion_tokens=500)
prompt_cost, completion_cost = dashscope_cost_per_token(
model="qwen-max", usage=usage
)
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-max", usage=usage)
model_info = litellm.get_model_info("dashscope/qwen-max")
expected_prompt_cost = 1000 * model_info["input_cost_per_token"]
@ -527,9 +544,7 @@ class TestDashscopeCostCalculator:
"""
# Tier 1 for qwen-flash is [0, 256,000] tokens
usage = Usage(prompt_tokens=100000, completion_tokens=50000)
prompt_cost, completion_cost = dashscope_cost_per_token(
model="qwen-flash", usage=usage
)
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage)
model_info = litellm.get_model_info("dashscope/qwen-flash")
tier_1_pricing = model_info["tiered_pricing"][0]
@ -547,9 +562,7 @@ class TestDashscopeCostCalculator:
"""
# Tiering for qwen-flash: Tier 1: [0, 256k], Tier 2: [256k, 1M]
usage = Usage(prompt_tokens=300000, completion_tokens=300000)
prompt_cost, completion_cost = dashscope_cost_per_token(
model="qwen-flash", usage=usage
)
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage)
model_info = litellm.get_model_info("dashscope/qwen-flash")
tier_1 = model_info["tiered_pricing"][0]
@ -561,9 +574,7 @@ class TestDashscopeCostCalculator:
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
graduated_prompt_cost = (256000 * tier_1["input_cost_per_token"]) + (
44000 * tier_2["input_cost_per_token"]
)
graduated_prompt_cost = (256000 * tier_1["input_cost_per_token"]) + (44000 * tier_2["input_cost_per_token"])
assert prompt_cost > graduated_prompt_cost
def test_dashscope_tiered_pricing_boundary_stays_in_lower_tier(self):
@ -572,18 +583,12 @@ class TestDashscopeCostCalculator:
official `0 < Token <= 256K` phrasing.
"""
usage = Usage(prompt_tokens=256000, completion_tokens=1000)
prompt_cost, completion_cost = dashscope_cost_per_token(
model="qwen-flash", usage=usage
)
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage)
tier_1 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][0]
assert math.isclose(
prompt_cost, 256000 * tier_1["input_cost_per_token"], rel_tol=1e-10
)
assert math.isclose(
completion_cost, 1000 * tier_1["output_cost_per_token"], rel_tol=1e-10
)
assert math.isclose(prompt_cost, 256000 * tier_1["input_cost_per_token"], rel_tol=1e-10)
assert math.isclose(completion_cost, 1000 * tier_1["output_cost_per_token"], rel_tol=1e-10)
def test_dashscope_tiered_pricing_output_uses_input_selected_tier(self):
"""
@ -595,9 +600,7 @@ class TestDashscopeCostCalculator:
tier_1 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][0]
assert math.isclose(
completion_cost, 400000 * tier_1["output_cost_per_token"], rel_tol=1e-10
)
assert math.isclose(completion_cost, 400000 * tier_1["output_cost_per_token"], rel_tol=1e-10)
def test_dashscope_tiered_pricing_with_caching(self):
"""
@ -626,17 +629,13 @@ class TestDashscopeCostCalculator:
"""
Requests above the highest declared range bill entirely at the last tier's rate.
"""
usage = Usage(
prompt_tokens=1200000, completion_tokens=1000
) # Max defined range for qwen-flash is 1M
usage = Usage(prompt_tokens=1200000, completion_tokens=1000) # Max defined range for qwen-flash is 1M
prompt_cost, _ = dashscope_cost_per_token(model="qwen-flash", usage=usage)
tier_2 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][1]
assert math.isclose(
prompt_cost, 1200000 * tier_2["input_cost_per_token"], rel_tol=1e-10
)
assert math.isclose(prompt_cost, 1200000 * tier_2["input_cost_per_token"], rel_tol=1e-10)
def _register_tiered_model(self, model_key: str, tiered_pricing: list[dict]) -> None:
litellm.model_cost[model_key] = {
@ -671,9 +670,7 @@ class TestDashscopeCostCalculator:
self._register_string_valued_tiered_model("dashscope/qwen-str-tier-test")
usage = Usage(prompt_tokens=500, completion_tokens=200)
prompt_cost, completion_cost = dashscope_cost_per_token(
model="qwen-str-tier-test", usage=usage
)
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-str-tier-test", usage=usage)
assert math.isclose(prompt_cost, 500 * float("4e-07"), rel_tol=1e-10)
assert math.isclose(completion_cost, 200 * float("1.6e-06"), rel_tol=1e-10)
@ -686,9 +683,7 @@ class TestDashscopeCostCalculator:
self._register_string_valued_tiered_model("dashscope/qwen-str-tier-test")
usage = Usage(prompt_tokens=2500, completion_tokens=3000)
prompt_cost, completion_cost = dashscope_cost_per_token(
model="qwen-str-tier-test", usage=usage
)
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-str-tier-test", usage=usage)
assert math.isclose(prompt_cost, 2500 * float("8e-07"), rel_tol=1e-10)
assert math.isclose(completion_cost, 3000 * float("3.2e-06"), rel_tol=1e-10)
@ -721,18 +716,12 @@ class TestDashscopeCostCalculator:
usage = Usage(
prompt_tokens=300000, # 200k new + 60k cache creation + 40k cache read
completion_tokens=1000,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=40000, cache_creation_tokens=60000
),
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=40000, cache_creation_tokens=60000),
)
prompt_cost, _ = dashscope_cost_per_token(
model="qwen-cache-write-test", usage=usage
)
prompt_cost, _ = dashscope_cost_per_token(model="qwen-cache-write-test", usage=usage)
expected_prompt_cost = (
(200000 * 6.5e-07) + (60000 * 8.125e-07) + (40000 * 6.5e-08)
)
expected_prompt_cost = (200000 * 6.5e-07) + (60000 * 8.125e-07) + (40000 * 6.5e-08)
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
@ -769,13 +758,9 @@ class TestDashscopeCostCalculator:
completion_tokens_details={"reasoning_tokens": 170},
)
prompt_cost, _ = dashscope_cost_per_token(
model="qwen-nested-cache-write-test", usage=usage
)
prompt_cost, _ = dashscope_cost_per_token(model="qwen-nested-cache-write-test", usage=usage)
assert math.isclose(
prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10
)
assert math.isclose(prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10)
def test_dashscope_tiered_cache_creation_falls_back_to_tier_input_rate(self):
"""
@ -799,9 +784,7 @@ class TestDashscopeCostCalculator:
prompt_tokens_details=PromptTokensDetailsWrapper(cache_creation_tokens=4000),
)
prompt_cost, _ = dashscope_cost_per_token(
model="qwen-no-cache-write-test", usage=usage
)
prompt_cost, _ = dashscope_cost_per_token(model="qwen-no-cache-write-test", usage=usage)
assert math.isclose(prompt_cost, 10000 * 3.25e-07, rel_tol=1e-10)
@ -819,18 +802,12 @@ class TestDashscopeCostCalculator:
usage = Usage(
prompt_tokens=10000,
completion_tokens=100,
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=2000, cache_creation_tokens=3000
),
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=2000, cache_creation_tokens=3000),
)
prompt_cost, _ = dashscope_cost_per_token(
model="qwen-flat-cache-write-test", usage=usage
)
prompt_cost, _ = dashscope_cost_per_token(model="qwen-flat-cache-write-test", usage=usage)
expected_prompt_cost = (
(5000 * 3.25e-07) + (3000 * 4.063e-07) + (2000 * 3.25e-08)
)
expected_prompt_cost = (5000 * 3.25e-07) + (3000 * 4.063e-07) + (2000 * 3.25e-08)
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
@ -847,9 +824,7 @@ class TestDashscopeCostCalculator:
}
usage = Usage(prompt_tokens=500, completion_tokens=200)
prompt_cost, completion_cost = dashscope_cost_per_token(
model="qwen-input-only-tier-test", usage=usage
)
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-input-only-tier-test", usage=usage)
assert math.isclose(prompt_cost, 500 * 4e-07, rel_tol=1e-10)
assert math.isclose(completion_cost, 200 * 1.6e-06, rel_tol=1e-10)
@ -872,13 +847,9 @@ class TestDashscopeCostCalculator:
completion_tokens=200,
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150),
)
_, completion_cost = dashscope_cost_per_token(
model="qwen-input-only-reasoning-test", usage=usage
)
_, completion_cost = dashscope_cost_per_token(model="qwen-input-only-reasoning-test", usage=usage)
assert math.isclose(
completion_cost, (50 * 1.6e-06) + (150 * 4e-06), rel_tol=1e-10
)
assert math.isclose(completion_cost, (50 * 1.6e-06) + (150 * 4e-06), rel_tol=1e-10)
def test_dashscope_tier_output_rate_wins_over_the_model_reasoning_rate(self):
"""
@ -903,9 +874,7 @@ class TestDashscopeCostCalculator:
completion_tokens=200,
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150),
)
_, completion_cost = dashscope_cost_per_token(
model="qwen-tier-output-reasoning-test", usage=usage
)
_, completion_cost = dashscope_cost_per_token(model="qwen-tier-output-reasoning-test", usage=usage)
assert math.isclose(completion_cost, 200 * 1.6e-06, rel_tol=1e-10)
@ -927,9 +896,7 @@ class TestDashscopeCostCalculator:
completion_tokens=200,
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150),
)
_, completion_cost = dashscope_cost_per_token(
model="qwen-zero-reasoning-test", usage=usage
)
_, completion_cost = dashscope_cost_per_token(model="qwen-zero-reasoning-test", usage=usage)
assert math.isclose(completion_cost, 50 * 1.6e-06, rel_tol=1e-10)
@ -956,9 +923,7 @@ class TestDashscopeCostCalculator:
completion_tokens=200,
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150),
)
_, completion_cost = dashscope_cost_per_token(
model="qwen-tier-zero-reasoning-test", usage=usage
)
_, completion_cost = dashscope_cost_per_token(model="qwen-tier-zero-reasoning-test", usage=usage)
assert math.isclose(completion_cost, 50 * 1.6e-06, rel_tol=1e-10)
@ -987,9 +952,7 @@ class TestDashscopeCostCalculator:
}
usage = Usage(prompt_tokens=0, completion_tokens=500)
prompt_cost, completion_cost = dashscope_cost_per_token(
model="qwen-zero-input-test", usage=usage
)
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-zero-input-test", usage=usage)
assert prompt_cost == 0.0
assert math.isclose(completion_cost, 500 * 1.6e-06, rel_tol=1e-10)

View file

@ -1,65 +1,76 @@
from collections.abc import Mapping
from copy import deepcopy
from datetime import datetime, timezone
from typing import Final
import pytest
from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate
from litellm.types.utils import ModelInfo
import litellm
from litellm.exceptions import ModelNotMappedError
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, get_billed_token_rates
from litellm.types.utils import Usage
def test_explicit_cache_read_rate_and_missing_input_rate_keep_the_entry_untouched() -> None:
explicit_info: ModelInfo = {"input_cost_per_token": 2e-6, "cache_read_input_token_cost": 1e-6}
no_input_rate_info: ModelInfo = {"output_cost_per_token": 3e-6}
assert with_default_cache_read_rate(explicit_info) is explicit_info
assert with_default_cache_read_rate(no_input_rate_info) is no_input_rate_info
@pytest.mark.parametrize(
("extra", "expected"),
(
({}, 0.0001),
({"cache_read_input_token_cost": None}, 0.0001),
({"cache_read_input_token_cost": 0.0}, 0.0),
({"cache_read_input_token_cost": 0.3e-6}, 0.00003),
({"off_peak_pricing": {"hours_utc": "14:00-00:00", "input_cost_per_token": 1e-6}}, 0.00005),
({"off_peak_pricing": {"hours_utc": "14:00-00:00", "output_cost_per_token": 3e-6}}, 0.0001),
({"off_peak_pricing": {"hours_utc": "14:00-00:00", "input_cost_per_token": "1e-6"}}, 0.00005),
(
{
"off_peak_pricing": {
"hours_utc": "14:00-00:00",
"input_cost_per_token": 1e-6,
"cache_read_input_token_cost": None,
}
},
0.0001,
),
({"tiered_pricing": [{"range": [0, 200], "input_cost_per_token": 4e-6}]}, 0.0004),
),
)
def test_fireworks_default_cache_rate_matches_projected_rates_without_mutating_the_row(
monkeypatch: pytest.MonkeyPatch,
extra: Mapping[str, object],
expected: float,
) -> None:
model: Final = "fireworks_ai/cache-default-test"
row: Final = {"input_cost_per_token": "2e-6", "litellm_provider": "fireworks_ai", "mode": "chat", **extra}
original: Final = deepcopy(row)
monkeypatch.setitem(litellm.model_cost, model, row)
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="fireworks_ai")
usage: Final = Usage(prompt_tokens=100, completion_tokens=0, prompt_tokens_details={"cached_tokens": 100})
when: Final = datetime(2026, 1, 1, 16, tzinfo=timezone.utc)
assert generic_cost_per_token(
model,
usage,
"fireworks_ai",
model_info=model_info,
current_time=when,
) == pytest.approx((expected, 0))
assert get_billed_token_rates(
model,
"fireworks_ai",
usage,
model_info=model_info,
current_time=when,
).cache_read_input_token_cost == pytest.approx(expected / 100)
assert row == original
def test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry() -> None:
model_info: ModelInfo = {
"input_cost_per_token": 2e-6,
"off_peak_pricing": {
"hours_utc": "14:00-00:00",
"input_cost_per_token": 1e-6,
"output_cost_per_token": 3e-6,
},
}
original: ModelInfo = deepcopy(model_info)
derived = with_default_cache_read_rate(model_info)
assert model_info == original
assert derived is not model_info
assert derived["cache_read_input_token_cost"] == pytest.approx(2e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO)
assert derived["off_peak_pricing"] == {
"hours_utc": "14:00-00:00",
"input_cost_per_token": 1e-6,
"output_cost_per_token": 3e-6,
"cache_read_input_token_cost": 1e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO,
}
def test_off_peak_window_without_its_own_input_rate_reuses_the_standard_derived_rate() -> None:
model_info: ModelInfo = {
"input_cost_per_token": 2e-6,
"off_peak_pricing": {"hours_utc": "14:00-00:00", "output_cost_per_token": 3e-6},
}
derived = with_default_cache_read_rate(model_info)
assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == derived["cache_read_input_token_cost"]
def test_string_rates_from_config_are_coerced_before_the_discount_is_applied() -> None:
model_info: ModelInfo = {
"input_cost_per_token": "2e-6",
"off_peak_pricing": {
"hours_utc": "14:00-00:00",
"input_cost_per_token": "1e-6",
},
}
derived = with_default_cache_read_rate(model_info)
assert derived["cache_read_input_token_cost"] == pytest.approx(1e-6)
assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == pytest.approx(5e-7)
def test_fireworks_missing_input_rate_remains_unknown_without_a_cache_price(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "fireworks_ai/cache-unknown-test"
row: Final = {"litellm_provider": "fireworks_ai", "mode": "chat", "output_cost_per_token": 3e-6}
monkeypatch.setitem(litellm.model_cost, model, row)
with pytest.raises(ModelNotMappedError):
generic_cost_per_token(
model,
Usage(prompt_tokens=100, completion_tokens=0, prompt_tokens_details={"cached_tokens": 100}),
"fireworks_ai",
)
assert row == {"litellm_provider": "fireworks_ai", "mode": "chat", "output_cost_per_token": 3e-6}

View file

@ -323,10 +323,8 @@ def test_an_entry_without_an_input_rate_gets_no_cache_read_fallback():
}
usage: Final = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200)
prompt_cost, completion_cost = cost_per_token(model="accounts/fireworks/models/no-input-rate-test", usage=usage)
assert prompt_cost == 0
assert completion_cost == 200 * 2e-06
with pytest.raises(litellm.ModelNotMappedError, match="input_cost_per_token"):
cost_per_token(model="accounts/fireworks/models/no-input-rate-test", usage=usage)
ROUTED_MODEL: Final = next(

View file

@ -7,8 +7,12 @@ per the MCP spec.
"""
from types import SimpleNamespace
from typing import Final
from unittest.mock import patch
import pytest
import litellm
from litellm.proxy._experimental.mcp_server.sampling_handler import (
_has_priorities,
_resolve_model_from_preferences,
@ -30,6 +34,14 @@ def _prefs(*, hints=None, cost=None, speed=None, intelligence=None):
)
def test_priority_reader_tolerates_missing_token_prices(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "openai/m5-sampling-reader"
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat"})
info: Final = litellm.get_model_info(model)
assert (info["input_cost_per_token"], info["output_cost_per_token"]) == (None, None)
assert _select_model_by_priority([model], _prefs(cost=1.0)) == model
# Model info stubs keyed by model name
_MODEL_INFO = {
"gpt-3.5-turbo": {

View file

@ -8,17 +8,77 @@ See: https://github.com/BerriAI/litellm/issues/24770
"""
import copy
from collections.abc import Mapping
from typing import Final
import pytest
import litellm
from litellm.proxy.auth.auth_checks import is_model_cost_zero
from litellm._catalog import is_priced
from litellm.proxy.auth.auth_checks import is_model_cost_zero, model_has_no_cost_mapping
from litellm.router import Router
UNPRICED_ZERO_COST_MODEL = "ollama/unpriced-zero-cost-target"
@pytest.mark.parametrize(
("prices", "present", "free"),
(
({}, False, False),
({"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, True, True),
({"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}, True, False),
({"input_cost_per_token": 0.0}, True, False),
({"output_cost_per_token": 2.0}, True, False),
({"cache_read_input_token_cost": 0.0}, True, False),
(
{
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"ptu_count": 10,
"cost_per_ptu_per_hour": 1.0,
},
True,
False,
),
(
{
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
"tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 3.0, "output_cost_per_token": 4.0}],
},
True,
False,
),
),
ids=(
"metadata-only",
"explicit-free",
"paid",
"input-only-zero",
"output-only-positive",
"cache-only-zero",
"ptu",
"flat-plus-tier",
),
)
def test_native_price_presence_does_not_replace_free_budget_policy(
monkeypatch: pytest.MonkeyPatch, prices: Mapping[str, object], present: bool, free: bool
) -> None:
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
router: Final = Router(
model_list=[
{
"model_name": "m5-budget-group",
"litellm_params": {"model": "openai/m5-budget-backend", "api_key": "fake-key"},
"model_info": {"id": "m5-budget-deployment", **prices},
}
]
)
assert is_priced("m5-budget-deployment") is present
assert model_has_no_cost_mapping("m5-budget-group", router) is not present
assert is_model_cost_zero("m5-budget-group", router) is free
def _explicitly_free(name: str, model: str = "gpt-3.5-turbo", **model_info: float) -> dict:
return {
"model_name": name,

View file

@ -1,6 +1,7 @@
from __future__ import annotations
import math
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from typing import Final
@ -25,6 +26,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
)
from litellm.proxy.spend_tracking.budget_reservation import (
_get_team_member_budget_counter,
estimate_request_input_cost,
estimate_request_max_cost,
get_budget_window_start,
release_unbound_budget_reservation,
@ -197,6 +199,68 @@ def _tiered_deployment(input_cost_per_token: float) -> Deployment:
TIERED_BODY: Final = {"model": "tiered-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}
@pytest.mark.parametrize(
("prices", "expected_input", "expected_max"),
(
({}, None, None),
({"input_cost_per_token": None, "output_cost_per_token": None}, None, None),
({"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, 0.0, 0.0),
({"input_cost_per_token": 2.0}, 20.0, None),
),
ids=("missing", "null", "explicit-zero", "input-only"),
)
def test_reservation_reader_preserves_unknown_token_prices(
monkeypatch: pytest.MonkeyPatch,
prices: Mapping[str, float | None],
expected_input: float | None,
expected_max: float | None,
) -> None:
model: Final = "openai/m5-reservation-reader"
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **prices})
request: Final = {"model": model, "messages": [{"role": "user", "content": "hello"}], "max_tokens": 5}
assert estimate_request_input_cost(request, "/chat/completions", None, {model: 10}) == expected_input
assert estimate_request_max_cost(request, "/chat/completions", None, {model: 10}) == expected_max
@pytest.mark.parametrize("tiered", (False, True))
def test_reservation_keeps_the_reasoning_max_and_output_multiplier_without_discounts(
monkeypatch: pytest.MonkeyPatch, tiered: bool
) -> None:
model: Final = "openai/m5-conservative-reservation"
prices: Final = {
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
"output_cost_per_reasoning_token": 5.0,
}
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": "openai",
"mode": "chat",
**({"tiered_pricing": [{"range": [0, 100], **prices}]} if tiered else prices),
"input_cost_per_token_flex": 0.1,
"output_cost_per_token_flex": 0.2,
"off_peak_pricing": {
"hours_utc": "00:00-00:00",
"input_cost_per_token": 0.01,
"output_cost_per_token": 0.02,
"output_cost_per_reasoning_token": 0.05,
},
},
)
request: Final = {
"model": model,
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 5,
"service_tier": "flex",
"n": 2,
"best_of": 3,
}
assert estimate_request_input_cost(request, "/chat/completions", None, {model: 10}) == 10.0
assert estimate_request_max_cost(request, "/chat/completions", None, {model: 10}) == 85.0
def test_repeated_estimates_reuse_cached_model_cost_info() -> None:
router: Final = Router(model_list=[_tiered_deployment(1e-06).model_dump()])
first: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router)

View file

@ -1,3 +1,5 @@
from collections.abc import Mapping
from datetime import datetime, timezone
from typing import Final, Literal
import pytest
@ -137,6 +139,81 @@ def test_compression_savings_priced_at_input_rate():
assert result.prompt_caching == 0.0
@pytest.mark.parametrize(
("prices", "saved_tokens", "expected"),
(
({}, 10, 0.0),
({"input_cost_per_token": None}, 10, 0.0),
({"input_cost_per_token": 0.0}, 10, 0.0),
({"input_cost_per_token": 2.0}, -10, 0.0),
({"input_cost_per_token": 2.0}, 10, 20.0),
),
)
def test_compression_keeps_the_flat_reporting_estimate_instead_of_token_selection(
monkeypatch: pytest.MonkeyPatch, prices: Mapping[str, float | None], saved_tokens: int, expected: float
) -> None:
model: Final = "openai/compression-policy-test"
monkeypatch.setitem(
litellm.model_cost,
model,
{
"litellm_provider": "openai",
"mode": "chat",
**prices,
"output_cost_per_token": 1.0,
"tiered_pricing": [{"range": [0, 2000], "input_cost_per_token": 0.1, "output_cost_per_token": 0.1}],
"input_cost_per_token_priority": 0.01,
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 0.001},
},
)
result: Final = compute_savings_spend(
model,
"openai",
saved_tokens,
False,
usage_object={"prompt_tokens": 1000, "completion_tokens": 0},
cost_breakdown={"service_tier": "priority"},
billed_at=datetime(2026, 1, 1, tzinfo=timezone.utc),
)
assert result.compression == expected
assert result.prompt_caching == 0.0
def test_compression_reporting_does_not_leak_deployment_prices(monkeypatch: pytest.MonkeyPatch) -> None:
model: Final = "openai/compression-policy-test"
row: Final = {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1.0,
"output_cost_per_token": 1.0,
}
monkeypatch.setitem(litellm.model_cost, model, dict(row))
router: Final = Router(
model_list=[
{
"model_name": "compression-a",
"litellm_params": {"model": model, "input_cost_per_token": 2.0},
"model_info": {"id": "compression-a"},
},
{
"model_name": "compression-b",
"litellm_params": {"model": model, "input_cost_per_token": 3.0},
"model_info": {"id": "compression-b"},
},
]
)
def estimate(model_id: str) -> float:
return compute_savings_spend(
model, "openai", 10, False, model_id=model_id, llm_router=lambda: router
).compression
assert estimate("compression-a") == 20.0
assert estimate("compression-b") == 30.0
assert estimate("compression-a") == 20.0
assert litellm.model_cost[model] == row
def test_prompt_caching_savings_priced_at_input_minus_cache_read():
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")
# A model that supports prompt caching must charge less to read from cache;
@ -635,8 +712,6 @@ def test_equal_modeled_usage_is_zero_under_equivalent_model_names() -> None:
assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage, usage) == 0.0
def test_unresolvable_baseline_remains_unknown():
usage = _usage(fresh=2000, cached=0, written=0, out=500)
assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) is None

View file

@ -13,8 +13,11 @@ from openai.types.completion_usage import CompletionUsage
from pydantic import BaseModel
import litellm
from litellm.exceptions import ModelNotMappedError
from litellm import TranscriptionResponse, model_cost
from litellm.constants import REALTIME_METERED_USAGE_KEY
from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
from litellm.integrations.custom_logger import CustomLogger
@ -43,6 +46,30 @@ import threading
from unittest.mock import MagicMock, patch
@pytest.mark.parametrize("provider", (None, "", 123, {}))
def test_registered_bedrock_pricing_does_not_replace_a_declared_provider(
monkeypatch: pytest.MonkeyPatch, provider: object
) -> None:
model: Final = "bedrock-invalid-provider-context"
litellm.register_model(
{model: {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}},
custom_llm_provider="bedrock",
persist_across_reloads=False,
)
monkeypatch.setitem(litellm.model_cost[model], "litellm_provider", provider)
if provider == "":
assert litellm.cost_per_token(model, 10, 20, custom_llm_provider="bedrock") == (10.0, 40.0)
assert litellm.get_model_info(model=model)["litellm_provider"] == ""
else:
with pytest.raises(ModelNotMappedError, match="litellm_provider"):
litellm.cost_per_token(model, 10, 20, custom_llm_provider="bedrock")
assert litellm.model_cost[model] == {
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
"litellm_provider": provider,
}
class ResponseCostCapture(CustomLogger):
def __init__(self) -> None:
self.success_cost: float | None = None
@ -493,7 +520,7 @@ def test_completion_cost_skips_huggingface_config_request_with_custom_pricing(
assert cost == pytest.approx(usage.prompt_tokens * 0.001 + usage.completion_tokens * 0.002)
def test_completion_cost_returns_zero_for_capability_only_generalization(
def test_completion_cost_rejects_unknown_pricing_for_capability_only_generalization(
_local_model_cost_map: None,
_claude_family_generalization: None,
monkeypatch: pytest.MonkeyPatch,
@ -506,13 +533,8 @@ def test_completion_cost_returns_zero_for_capability_only_generalization(
usage=Usage(prompt_tokens=17, completion_tokens=9, total_tokens=26),
)
cost: Final = completion_cost(
completion_response=response,
model=model,
custom_llm_provider="bedrock",
)
assert cost == 0.0
with pytest.raises(ModelNotMappedError, match="input_cost_per_token"):
completion_cost(completion_response=response, model=model, custom_llm_provider="bedrock")
@pytest.mark.parametrize("custom_llm_provider", ["openai", "hosted_vllm"])
@ -1842,7 +1864,15 @@ def test_cost_calculator_with_cache_creation():
}
),
)
model = "claude-sonnet-4@20250514"
model: Final = "claude-cache-creation-contract"
litellm.register_model({f"vertex_ai/{model}": {
"litellm_provider": "vertex_ai",
"mode": "chat",
"input_cost_per_token": 0.000002,
"output_cost_per_token": 0.000003,
"cache_read_input_token_cost": 0.0000001,
"cache_creation_input_token_cost": 0.000004,
}})
assert litellm_model_response.usage.prompt_tokens_details.cached_tokens == 28491
@ -1852,7 +1882,7 @@ def test_cost_calculator_with_cache_creation():
custom_llm_provider="vertex_ai",
)
print(result)
assert result == pytest.approx(13 * 0.000003 + 28491 * 0.0000001 + 15 * 0.000004)
def test_bedrock_cost_calculator_comparison_with_without_cache():
@ -7350,6 +7380,45 @@ def test_vertex_ai_medlm_completion_cost():
assert predictive_cost > 0
@pytest.mark.parametrize(
("direction", "threshold_field", "prompt_tokens", "completion_tokens", "expected"),
[
("output", None, 200001, 10, (20.0, 30.0)),
("output", "output_cost_per_token_above_200k_tokens", 200001, 10, (20.0, 50.0)),
("output", "output_cost_per_token_above_128k_tokens", 200001, 10, (20.0, 30.0)),
("output", "output_cost_per_token_above_128k_tokens", 10, 128001, (20.0, 640005.0)),
("input", None, 10, 200001, (30.0, 20.0)),
("input", "input_cost_per_token_above_200k_tokens", 200001, 10, (1000005.0, 20.0)),
("input", "input_cost_per_token_above_128k_tokens", 200001, 10, (1000005.0, 20.0)),
],
)
def test_vertex_mixed_character_pricing_only_prices_the_used_token_direction(
direction: Literal["input", "output"],
threshold_field: str | None,
prompt_tokens: int,
completion_tokens: int,
expected: tuple[float, float],
) -> None:
model: Final = "vertex_ai/mixed-character-contract"
character_direction: Final = "input" if direction == "output" else "output"
litellm.register_model({model: {
"litellm_provider": "vertex_ai",
"mode": "chat",
f"{character_direction}_cost_per_character": 2.0,
f"{direction}_cost_per_token": 3.0,
**({threshold_field: 5.0} if threshold_field is not None else {}),
}})
assert cost_per_token(
model=model,
custom_llm_provider="vertex_ai",
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
prompt_characters=10 if character_direction == "input" else None,
completion_characters=10 if character_direction == "output" else None,
) == expected
def test_vertex_ai_embedding_completion_cost(caplog):
"""
Relevant issue - https://github.com/BerriAI/litellm/issues/4630

View file

@ -1,26 +1,16 @@
"""
Test that register_model() in completion() and embedding() passes all
custom pricing fields from kwargs and model_info, not just the base
input/output costs.
Previously, only input_cost_per_token, output_cost_per_token, and
litellm_provider were forwarded. Fields like cache_read_input_token_cost,
mode, and supports_prompt_caching were dropped, causing incorrect cost
calculations for DB-sourced models with prompt caching pricing.
"""
import copy
import itertools
import os
from collections.abc import Callable, Mapping
from datetime import datetime, timezone
from queue import Queue
from typing import Final, NoReturn
import pytest
import litellm
from litellm._internal_context import pinned_billing_time
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map
from litellm.main import _build_custom_pricing_entry
from litellm.utils import _invalidate_model_cost_lowercase_map
@ -28,6 +18,350 @@ def _snapshot_model_cost_entries(keys):
return {key: copy.deepcopy(litellm.model_cost.get(key)) for key in keys}
@pytest.mark.parametrize("stream", (False, True))
def test_sdk_custom_prices_do_not_leak_between_tenants_or_into_deployments(
monkeypatch: pytest.MonkeyPatch, stream: bool
) -> None:
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
model: Final = "openai/m5-tenant-backend"
deployment_id: Final = "m5-tenant-deployment"
catalog_row: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0}
owned_row: Final = {"litellm_provider": "openai", "input_cost_per_token": 7.0, "output_cost_per_token": 8.0}
monkeypatch.setitem(litellm.model_cost, model, catalog_row.copy())
monkeypatch.setitem(litellm.model_cost, deployment_id, owned_row.copy())
events: Final[Queue[tuple[object, object]]] = Queue()
def on_success(kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime) -> None:
if kwargs.get("stream") and kwargs.get("complete_streaming_response") is None:
return
events.put((kwargs.get("response_cost"), kwargs.get("complete_streaming_response") or response))
monkeypatch.setattr(litellm, "success_callback", [on_success])
for input_rate, output_rate, per_call in (
(5.0, 6.0, {}),
(
1.0,
4.0,
{
"input_cost_per_token": 1.0,
"output_cost_per_token": 4.0,
"metadata": {"model_info": {"id": deployment_id}},
},
),
(5.0, 6.0, {}),
):
result: Final = litellm.completion(
model=model,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
mock_response="Hello back",
stream=stream,
**per_call,
)
if isinstance(result, CustomStreamWrapper):
assert tuple(result)
cost, response = events.get(timeout=10)
assert isinstance(response, litellm.ModelResponse)
assert isinstance(response.usage, litellm.Usage)
expected: Final = response.usage.prompt_tokens * input_rate + response.usage.completion_tokens * output_rate
assert cost == pytest.approx(expected)
if not stream:
assert response._hidden_params["response_cost"] == pytest.approx(expected)
assert litellm.model_cost[model] == catalog_row
assert litellm.model_cost[deployment_id] == owned_row
assert litellm.cost_per_token(deployment_id, 10, 20, custom_llm_provider="openai") == pytest.approx((70.0, 160.0))
@pytest.mark.parametrize(
("prices", "usage", "service_tier", "expected"),
(
(
{"output_cost_per_reasoning_token": 3.0},
litellm.Usage(prompt_tokens=10, completion_tokens=10, completion_tokens_details={"reasoning_tokens": 5}),
"default",
35.0,
),
(
{"input_cost_per_audio_token": 3.0, "output_cost_per_audio_token": 4.0},
litellm.Usage(
prompt_tokens=10,
completion_tokens=10,
prompt_tokens_details={"text_tokens": 6, "audio_tokens": 4},
completion_tokens_details={"text_tokens": 6, "audio_tokens": 4},
),
"default",
46.0,
),
(
{"input_cost_per_token_priority": 3.0, "output_cost_per_token_priority": 4.0},
litellm.Usage(prompt_tokens=10, completion_tokens=10),
"priority",
70.0,
),
),
ids=("reasoning", "audio", "priority"),
)
def test_sdk_request_prices_preserve_non_base_token_fields_without_catalog_writes(
monkeypatch: pytest.MonkeyPatch,
prices: Mapping[str, float],
usage: litellm.Usage,
service_tier: str,
expected: float,
) -> None:
model: Final = "openai/m5-request-token-fields"
row: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0}
monkeypatch.setitem(litellm.model_cost, model, row.copy())
response: Final = litellm.completion(
model=model,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
input_cost_per_token=1.0,
output_cost_per_token=2.0,
mock_response=litellm.ModelResponse(model=model, usage=usage, service_tier=service_tier),
**prices,
)
assert response._hidden_params["response_cost"] == pytest.approx(expected)
assert litellm.model_cost[model] == row
assert litellm.cost_per_token(model, 10, 10, custom_llm_provider="openai") == pytest.approx((50.0, 60.0))
@pytest.mark.parametrize("stream", (False, True))
def test_router_request_prices_override_only_the_current_call(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None:
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
model: Final = "openai/m5-routed-request"
deployment_id: Final = "m5-routed-request-deployment"
catalog: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0}
monkeypatch.setitem(litellm.model_cost, model, catalog.copy())
events: Final = Queue[tuple[object, object]]()
class Costs(CustomLogger):
def log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
events.put((kwargs.get("response_cost"), kwargs.get("complete_streaming_response", response_obj)))
monkeypatch.setattr(litellm, "callbacks", [Costs()])
router: Final = litellm.Router(
model_list=[
{
"model_name": "m5-request-group",
"litellm_params": {"model": model, "api_key": "fake-key"},
"model_info": {
"id": deployment_id,
"input_cost_per_token": 7.0,
"output_cost_per_token": 8.0,
},
}
],
num_retries=0,
)
owned: Final = copy.deepcopy(litellm.model_cost[deployment_id])
try:
for input_rate, output_rate, per_call in (
(7.0, 8.0, {}),
(1.0, 2.0, {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}),
(0.0, 0.0, {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}),
(7.0, 8.0, {}),
):
result: Final = router.completion(
model="m5-request-group",
messages=[{"role": "user", "content": "hello"}],
mock_response="Hello back",
stream=stream,
**per_call,
)
if isinstance(result, CustomStreamWrapper):
assert tuple(result)
cost, response = events.get(timeout=10)
assert isinstance(response, litellm.ModelResponse)
assert isinstance(response.usage, litellm.Usage)
assert cost == pytest.approx(
response.usage.prompt_tokens * input_rate + response.usage.completion_tokens * output_rate
)
assert litellm.model_cost[deployment_id] == owned
assert litellm.model_cost[model] == catalog
finally:
router.discard()
def test_request_prices_reach_callback_cache_and_reasoning_breakdown_without_registration(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.integrations.custom_logger import CustomLogger
model: Final = "openai/m5-request-breakdown"
row: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0}
monkeypatch.setitem(litellm.model_cost, model, row.copy())
events: Final = Queue[object]()
class Costs(CustomLogger):
def log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
events.put(kwargs.get("standard_logging_object"))
monkeypatch.setattr(litellm, "callbacks", [Costs()])
response: Final = litellm.completion(
model=model,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
input_cost_per_token=1.0,
output_cost_per_token=2.0,
cache_read_input_token_cost=0.1,
cache_creation_input_token_cost=1.25,
output_cost_per_reasoning_token=3.0,
model_info={"mode": "chat", "supports_prompt_caching": True, "max_tokens": 8192},
mock_response=litellm.ModelResponse(
model=model,
usage=litellm.Usage(
prompt_tokens=100,
completion_tokens=10,
prompt_tokens_details={"cached_tokens": 60, "cache_creation_tokens": 20},
completion_tokens_details={"reasoning_tokens": 5},
),
),
)
payload: Final = events.get(timeout=10)
assert isinstance(payload, dict)
assert payload["response_cost"] == pytest.approx(76.0)
assert payload["cost_breakdown"] == pytest.approx(
{
"input_cost": 51.0,
"output_cost": 25.0,
"total_cost": 76.0,
"tool_usage_cost": 0.0,
"service_tier": None,
"data_residency": None,
"vertex_location": None,
"cache_read_cost": 6.0,
"cache_creation_cost": 25.0,
"reasoning_cost": 15.0,
"original_cost": 76.0,
"discount_percent": 0.0,
"discount_amount": 0.0,
"margin_percent": 0.0,
"margin_fixed_amount": 0.0,
"margin_total_amount": 0.0,
}
)
assert response._hidden_params["response_cost"] == pytest.approx(76.0)
assert litellm.model_cost[model] == row
@pytest.mark.parametrize(("cache_price", "expected"), ((None, 0.5), (0.0, 0.0), (0.25, 0.25)))
def test_request_all_cache_usage_retains_unused_rates_and_zero_overrides(
monkeypatch: pytest.MonkeyPatch, cache_price: float | None, expected: float
) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
model: Final = "openai/m5-request-all-cache"
row: Final = {
"litellm_provider": "openai",
"input_cost_per_token": 5.0,
"output_cost_per_token": 6.0,
"cache_read_input_token_cost": 0.5,
}
monkeypatch.setitem(litellm.model_cost, model, row.copy())
when: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
logging_obj: Final = Logging(
model=model,
messages=[{"role": "user", "content": "hello"}],
stream=False,
call_type="completion",
start_time=when,
litellm_call_id="m5-request-all-cache",
function_id="f",
kwargs={
"input_cost_per_token": 1.0,
"output_cost_per_token": 2.0,
"cache_read_input_token_cost": cache_price,
},
)
with pinned_billing_time(when):
total: Final = litellm.completion_cost(
model=model,
custom_llm_provider="openai",
custom_pricing=True,
litellm_logging_obj=logging_obj,
completion_response=litellm.ModelResponse(
model=model,
usage=litellm.Usage(
prompt_tokens=100, completion_tokens=0, prompt_tokens_details={"cached_tokens": 100}
),
),
)
assert total == pytest.approx(100 * expected)
assert logging_obj.billed_token_rates is not None
assert logging_obj.billed_token_rates.input_cost_per_token == 1.0
assert logging_obj.billed_token_rates.output_cost_per_token == 2.0
assert logging_obj.billed_token_rates.cache_read_input_token_cost == expected
assert litellm.model_cost[model] == row
@pytest.mark.parametrize(("speed", "geo", "factor"), (("standard", "global", 1.0), ("fast", "us", 6.6)))
def test_ephemeral_request_prices_keep_provider_selected_adjustments(
monkeypatch: pytest.MonkeyPatch, speed: str, geo: str, factor: float
) -> None:
model: Final = "anthropic/m5-request-adjustments"
row: Final = {
"litellm_provider": "anthropic",
"mode": "chat",
"input_cost_per_token": 5.0,
"output_cost_per_token": 6.0,
"provider_specific_entry": {"fast": 6.0, "us": 1.1},
}
monkeypatch.setitem(litellm.model_cost, model, copy.deepcopy(row))
response: Final = litellm.completion(
model=model,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
input_cost_per_token=1.0,
output_cost_per_token=2.0,
mock_response=litellm.ModelResponse(
model=model,
usage=litellm.Usage(prompt_tokens=10, completion_tokens=10, speed=speed, inference_geo=geo),
),
)
assert response._hidden_params["response_cost"] == pytest.approx(30.0 * factor)
assert litellm.model_cost[model] == row
@pytest.mark.parametrize("field", ("input_cost_per_second", "output_cost_per_second", "cost_per_second"))
@pytest.mark.parametrize("rate", (0.0, 0.02))
def test_request_duration_prices_keep_python_time_billing_without_registration(field: str, rate: float) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
model: Final = "openai/m5-request-duration"
logging_obj: Final = Logging(
model=model,
messages=[{"role": "user", "content": "hello"}],
stream=False,
call_type="completion",
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
litellm_call_id="m5-request-duration",
function_id="f",
kwargs={field: rate},
)
total: Final = litellm.completion_cost(
model=model,
custom_llm_provider="openai",
custom_pricing=True,
litellm_logging_obj=logging_obj,
total_time=1500.0,
completion_response=litellm.ModelResponse(
model=model, usage=litellm.Usage(prompt_tokens=10, completion_tokens=10)
),
)
assert total == pytest.approx(rate * 1.5)
assert logging_obj.cost_breakdown is not None
assert logging_obj.cost_breakdown["input_cost"] == pytest.approx(rate * 1.5)
assert logging_obj.cost_breakdown["output_cost"] == 0.0
assert model not in litellm.model_cost
def _restore_model_cost_entries(original_entries):
for key, value in original_entries.items():
if value is None:
@ -37,131 +371,6 @@ def _restore_model_cost_entries(original_entries):
_invalidate_model_cost_lowercase_map()
def test_build_custom_pricing_entry_includes_all_kwargs_fields():
"""All CustomPricingLiteLLMParams fields present in kwargs should be
included in the resulting entry dict."""
kwargs = {
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
"cache_read_input_token_cost": 0.00025,
"cache_creation_input_token_cost": 0.005,
"output_cost_per_reasoning_token": 0.01,
"input_cost_per_audio_token": 0.003,
"unrelated_kwarg": "should_be_ignored",
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
)
assert entry["litellm_provider"] == "openai"
assert entry["input_cost_per_token"] == 0.001
assert entry["output_cost_per_token"] == 0.002
assert entry["cache_read_input_token_cost"] == 0.00025
assert entry["cache_creation_input_token_cost"] == 0.005
assert entry["output_cost_per_reasoning_token"] == 0.01
assert entry["input_cost_per_audio_token"] == 0.003
assert "unrelated_kwarg" not in entry
def test_build_custom_pricing_entry_merges_model_info_metadata():
"""Fields from model_info (mode, supports_prompt_caching, max_tokens)
should be merged into the entry when present."""
kwargs = {
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
}
model_info = {
"id": "deployment-123",
"mode": "chat",
"supports_prompt_caching": True,
"max_tokens": 128000,
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
model_info=model_info,
)
assert entry["mode"] == "chat"
assert entry["supports_prompt_caching"] is True
assert entry["max_tokens"] == 128000
def test_build_custom_pricing_entry_setdefault_does_not_override_existing():
"""model_info uses setdefault, so it should not override a key that is
already present in the entry dict. Currently CustomPricingLiteLLMParams
and the model_info keys (mode, supports_prompt_caching, max_tokens) do
not overlap, but if they ever do, setdefault ensures the kwargs-sourced
value wins."""
kwargs = {
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
}
model_info = {
"mode": "chat",
"supports_prompt_caching": True,
"max_tokens": 128000,
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
model_info=model_info,
)
assert entry["mode"] == "chat"
assert entry["supports_prompt_caching"] is True
assert entry["max_tokens"] == 128000
# Verify setdefault behavior: if a model_info key already exists in
# the entry (e.g. from a future CustomPricingLiteLLMParams addition),
# setdefault must not overwrite it.
entry["mode"] = "embedding" # simulate pre-existing value
# Re-apply setdefault the same way _build_custom_pricing_entry does
entry.setdefault("mode", model_info["mode"])
assert entry["mode"] == "embedding" # must NOT revert to "chat"
def test_build_custom_pricing_entry_skips_none_values():
"""Fields with None values in kwargs should not be included."""
kwargs = {
"input_cost_per_token": 0.001,
"output_cost_per_token": None, # explicitly None
"cache_read_input_token_cost": None,
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
)
assert entry["input_cost_per_token"] == 0.001
assert "output_cost_per_token" not in entry
assert "cache_read_input_token_cost" not in entry
def test_build_custom_pricing_entry_handles_no_model_info():
"""Should work correctly when model_info is None."""
kwargs = {
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
model_info=None,
)
assert entry["litellm_provider"] == "openai"
assert entry["input_cost_per_token"] == 0.001
assert entry["output_cost_per_token"] == 0.002
assert "mode" not in entry
def test_register_model_receives_cache_pricing_fields():
"""End-to-end: when register_model is called with a full pricing entry,
the cache pricing fields should be present in litellm.model_cost."""
@ -192,23 +401,6 @@ def test_register_model_receives_cache_pricing_fields():
litellm.model_cost.pop(model_key, None)
def test_build_custom_pricing_entry_time_based():
"""Time-based pricing fields should be included correctly."""
kwargs = {
"input_cost_per_second": 0.01,
"output_cost_per_second": 0.02,
}
entry = _build_custom_pricing_entry(
custom_llm_provider="openai",
kwargs=kwargs,
)
assert entry["litellm_provider"] == "openai"
assert entry["input_cost_per_second"] == 0.01
assert entry["output_cost_per_second"] == 0.02
def test_register_model_strips_none_litellm_provider():
"""``get_model_info`` returns ``litellm_provider: None`` for deployments
registered without a provider (e.g. ``Router.add_deployment`` flows).
@ -598,6 +790,16 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
assert builtin_input_cost > 0
try:
litellm.register_model(
{
deployment_id: {
"litellm_provider": "openai",
"mode": "embedding",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
}
}
)
litellm.embedding(
model=shared_key,
input=["hello"],
@ -640,6 +842,16 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
assert builtin_input_cost != override_input_cost
try:
litellm.register_model(
{
deployment_id: {
"litellm_provider": "openai",
"mode": "embedding",
"input_cost_per_token": override_input_cost,
"output_cost_per_token": override_input_cost * 2,
}
}
)
response = litellm.embedding(
model=shared_key,
input=["hello"],
@ -674,6 +886,16 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing():
assert builtin_input_cost > 0
try:
litellm.register_model(
{
deployment_id: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
}
}
)
litellm.completion(
model=shared_key,
messages=[{"role": "user", "content": "hello"}],
@ -693,11 +915,7 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing():
_restore_model_cost_entries(snapshot)
def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key():
"""Direct SDK calls (no router deployment id in metadata) keep the legacy
behavior: custom pricing is registered under ``{provider}/{model}`` and the
request is costed with it.
"""
def test_embedding_direct_sdk_custom_pricing_does_not_register_shared_key():
model_key = "openai/lit3991-direct-sdk-embed-model"
override_input_cost = 3e-05
try:
@ -710,13 +928,18 @@ def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key():
mock_response=[0.1, 0.2],
)
assert litellm.model_cost[model_key]["input_cost_per_token"] == override_input_cost
assert model_key not in litellm.model_cost
assert response._hidden_params["response_cost"] == pytest.approx(10 * override_input_cost)
cost = litellm.completion_cost(
completion_response=response,
model=model_key,
custom_llm_provider="openai",
call_type="embedding",
custom_pricing=True,
custom_cost_per_token={
"input_cost_per_token": override_input_cost,
"output_cost_per_token": override_input_cost * 2,
},
)
assert cost == pytest.approx(10 * override_input_cost)
finally:
@ -1417,10 +1640,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing():
object stays untouched, and the caller's incoming nested dict is never
inserted by reference into the merged result.
"""
from litellm.utils import update_dictionary
from typing import cast
from litellm.utils import update_dictionary
existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"})
existing = {"off_peak_pricing": existing_nested}
incoming_nested = cast(
@ -1619,12 +1842,12 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing():
del router
def test_completion_registers_cost_per_second_pricing():
def test_completion_uses_ephemeral_cost_per_second_pricing():
model_key: Final = "openai/test-cost-per-second-registration"
original_entries: Final = _snapshot_model_cost_entries([model_key])
try:
litellm.completion(
response: Final = litellm.completion(
model=model_key,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
@ -1632,7 +1855,14 @@ def test_completion_registers_cost_per_second_pricing():
mock_response="hello back",
)
assert litellm.model_cost[model_key]["cost_per_second"] == 0.02
assert model_key not in litellm.model_cost
assert response._hidden_params["response_cost"] is not None
assert litellm.completion_cost(
litellm.ModelResponse(model=model_key),
model=model_key,
custom_cost_per_second=0.02,
total_time=2000,
) == pytest.approx(0.04)
finally:
_restore_model_cost_entries(original_entries)
@ -1686,9 +1916,7 @@ def test_register_model_scopes_builtin_match_to_the_given_provider():
_restore_model_cost_entries(model_cost_entries)
def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider():
from litellm.main import _register_custom_pricing_for_request
def test_per_request_custom_pricing_does_not_write_a_colliding_deployment_id():
deployment_id: Final = "baseten/zai-org/glm-5.2"
builtin_key: Final = "baseten/zai-org/GLM-5.2"
shared_key: Final = "openai/zai-org/GLM-5.2"
@ -1696,20 +1924,20 @@ def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_prov
builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key])
openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models)
try:
_register_custom_pricing_for_request(
model="zai-org/GLM-5.2",
custom_llm_provider="openai",
kwargs={
"input_cost_per_token": 0.00000096,
"output_cost_per_token": 0.00000302,
"metadata": {"model_info": {"id": deployment_id}},
},
model_info={"mode": "chat"},
response: Final = litellm.completion(
model=shared_key,
messages=[{"role": "user", "content": "hello"}],
api_key="fake-key",
input_cost_per_token=0.00000096,
output_cost_per_token=0.00000302,
metadata={"model_info": {"id": deployment_id}},
mock_response=litellm.ModelResponse(
model=shared_key, usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30)
),
)
entry: Final = litellm.model_cost[deployment_id]
assert entry["input_cost_per_token"] == 0.00000096
assert entry["output_cost_per_token"] == 0.00000302
assert response._hidden_params["response_cost"] == pytest.approx(10 * 0.00000096 + 20 * 0.00000302)
assert litellm.model_cost.get(deployment_id) == model_cost_entries[deployment_id]
assert litellm.model_cost[builtin_key] == builtin_row_before
finally:
_restore_model_cost_entries(model_cost_entries)

View file

@ -3161,12 +3161,10 @@ def test_inherit_builtin_tiered_output_rate_fills_the_backend_flat_rate():
def test_inherit_builtin_tiered_output_rate_never_stores_a_synthesized_zero():
"""
Regression: get_model_info reports output_cost_per_token 0 for a backend that
only publishes tiered rates (e.g. dashscope/qwen-flash), and storing that zero
would mark the deployment as explicitly priced free.
Missing flat prices must not mark a tier-priced backend as explicitly free.
"""
backend_info = litellm.get_model_info(model="qwen-flash", custom_llm_provider="dashscope")
assert backend_info["output_cost_per_token"] == 0
assert backend_info["output_cost_per_token"] is None
model_info = {"tiered_pricing": [{"range": [0, 3000], "input_cost_per_token": 3.25e-07}]}
Router._inherit_builtin_tiered_output_rate(

View file

@ -2445,7 +2445,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None:
@respx.mock
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
def test_completion_registers_test_owned_model_prices() -> None:
def test_completion_uses_request_prices_without_registering_them() -> None:
model: Final = "migration-owned-pricing-model"
messages: Final = [{"role": "user", "content": "price control"}]
litellm.register_model(
@ -2488,8 +2488,7 @@ def test_completion_registers_test_owned_model_prices() -> None:
)
assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages}
assert litellm.model_cost[f"openai/{model}"]["input_cost_per_token"] == 0.001
assert litellm.model_cost[f"openai/{model}"]["output_cost_per_token"] == 0.002
assert f"openai/{model}" not in litellm.model_cost
assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001
assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002
assert response._hidden_params["response_cost"] == pytest.approx(0.018)