mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-19 09:53:33 +00:00
refactor(cost): move SDK token pricing to native engine
This commit is contained in:
parent
7479e494b0
commit
ce5ded9bf2
33 changed files with 2566 additions and 2624 deletions
|
|
@ -15,6 +15,12 @@ class Reloaded:
|
|||
models_count: int
|
||||
|
||||
|
||||
def is_priced(model: str, provider: str | None = None) -> bool:
|
||||
from litellm._catalog import _read
|
||||
|
||||
return _read.is_priced(model, provider)
|
||||
|
||||
|
||||
def reload_source(url: str) -> Source:
|
||||
if not url or is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
||||
return Bundled()
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from functools import wraps
|
|||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Final, Literal, ParamSpec, TypeVar, overload
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._catalog import _install
|
||||
from litellm.rust_bridge import _native
|
||||
|
|
@ -154,6 +154,27 @@ def model_info(key: str) -> dict[str, object]:
|
|||
return {name: raw[name] for name in result.fields}
|
||||
|
||||
|
||||
def is_priced(model: str, provider: str | None = None) -> bool:
|
||||
resolved: Final = resolve(model, provider)
|
||||
if not isinstance(resolved, _native.ResolvedModel):
|
||||
return False
|
||||
try:
|
||||
row: Final = _ROW.validate_python(
|
||||
dict[str, object].get(
|
||||
_install._live_rows(), # pyright: ignore[reportPrivateUsage] # presence uses the selected live row
|
||||
resolved.key,
|
||||
)
|
||||
)
|
||||
except ValidationError:
|
||||
return False
|
||||
contextual_row: Final = row if "litellm_provider" in row else {"litellm_provider": provider or "custom", **row}
|
||||
match _native.has_token_prices(contextual_row, policy="sdk_tokens"):
|
||||
case bool() as present:
|
||||
return present
|
||||
case _native.RowErrors():
|
||||
return False
|
||||
|
||||
|
||||
def supports(fields: Mapping[str, object], capability: str) -> bool | None:
|
||||
value: Final = fields.get(capability)
|
||||
return None if value is None else value is True
|
||||
|
|
|
|||
23
litellm/_cost/__init__.py
Normal file
23
litellm/_cost/__init__.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
from litellm._cost._price import (
|
||||
billing_context,
|
||||
cache_read_meter,
|
||||
cache_write_quantities,
|
||||
evaluate,
|
||||
price,
|
||||
request_model_info,
|
||||
request_prices,
|
||||
require_priced,
|
||||
token_usage,
|
||||
)
|
||||
|
||||
__all__ = (
|
||||
"billing_context",
|
||||
"cache_read_meter",
|
||||
"cache_write_quantities",
|
||||
"evaluate",
|
||||
"price",
|
||||
"request_model_info",
|
||||
"request_prices",
|
||||
"require_priced",
|
||||
"token_usage",
|
||||
)
|
||||
252
litellm/_cost/_price.py
Normal file
252
litellm/_cost/_price.py
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime, timezone
|
||||
from typing import (
|
||||
Final,
|
||||
Literal,
|
||||
cast, # noqa: TID251 # validated request prices extend the read-only public ModelInfo shape
|
||||
)
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.exceptions import ModelNotMappedError
|
||||
from litellm.rust_bridge import _native
|
||||
from litellm.types.utils import CacheCreationTokenDetails, CustomPricingLiteLLMParams, ModelInfo, Usage
|
||||
|
||||
_REQUEST_ROW: Final = TypeAdapter(dict[str, object])
|
||||
_REQUEST_PRICE_KEYS: Final = frozenset(CustomPricingLiteLLMParams.model_fields)
|
||||
|
||||
|
||||
def _request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
info: Final = kwargs.get("model_info")
|
||||
if not isinstance(info, Mapping):
|
||||
return {}
|
||||
fields: Final = _REQUEST_ROW.validate_python(info)
|
||||
return {
|
||||
key: value
|
||||
for key, value in fields.items()
|
||||
if key in ("mode", "supports_prompt_caching", "max_tokens") and value is not None
|
||||
}
|
||||
|
||||
|
||||
def request_prices(kwargs: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
has_tokens: Final = (
|
||||
kwargs.get("input_cost_per_token") is not None and kwargs.get("output_cost_per_token") is not None
|
||||
)
|
||||
has_seconds: Final = any(
|
||||
kwargs.get(key) is not None for key in ("cost_per_second", "input_cost_per_second", "output_cost_per_second")
|
||||
)
|
||||
if not has_tokens and not has_seconds:
|
||||
return None
|
||||
prices: Final = CustomPricingLiteLLMParams.model_validate(
|
||||
{key: value for key, value in kwargs.items() if key in _REQUEST_PRICE_KEYS and value is not None}
|
||||
)
|
||||
return {
|
||||
**_request_metadata(kwargs),
|
||||
**_REQUEST_ROW.validate_python(prices.model_dump(exclude_none=True, exclude_unset=True)),
|
||||
}
|
||||
|
||||
|
||||
def request_model_info(model: str, provider: str | None, overrides: Mapping[str, object]) -> ModelInfo:
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
def live_info() -> ModelInfo:
|
||||
try:
|
||||
return get_model_info(model=model, custom_llm_provider=provider)
|
||||
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unknown custom model
|
||||
return {
|
||||
"key": model,
|
||||
"litellm_provider": provider or "custom",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": None,
|
||||
"output_cost_per_token": None,
|
||||
"max_tokens": None,
|
||||
"max_input_tokens": None,
|
||||
"max_output_tokens": None,
|
||||
"supported_openai_params": None,
|
||||
}
|
||||
|
||||
return cast( # cast-ok: validated pricing keys preserve extra request meters
|
||||
ModelInfo, {**live_info(), **overrides}
|
||||
)
|
||||
|
||||
|
||||
def price(
|
||||
row: Mapping[str, object],
|
||||
usage: _native.UsageInput,
|
||||
when: _native.WhenInput,
|
||||
*,
|
||||
factors: Sequence[float] = (),
|
||||
cache_read_default_ratio: float | None = None,
|
||||
directional_token_threshold: int | None = None,
|
||||
) -> _native.Priced | _native.Unpriced | _native.RowErrors:
|
||||
if not factors:
|
||||
return _native.price(
|
||||
dict(row),
|
||||
usage,
|
||||
when,
|
||||
policy="sdk_tokens",
|
||||
cache_read_default_ratio=cache_read_default_ratio,
|
||||
directional_token_threshold=directional_token_threshold,
|
||||
)
|
||||
result: Final = evaluate(
|
||||
row,
|
||||
usage,
|
||||
when,
|
||||
factors=factors,
|
||||
cache_read_default_ratio=cache_read_default_ratio,
|
||||
directional_token_threshold=directional_token_threshold,
|
||||
)
|
||||
return result if isinstance(result, _native.RowErrors) else result.pricing
|
||||
|
||||
|
||||
def evaluate(
|
||||
row: Mapping[str, object],
|
||||
usage: _native.UsageInput,
|
||||
when: _native.WhenInput,
|
||||
*,
|
||||
project: Sequence[str] = (),
|
||||
factors: Sequence[float] = (),
|
||||
cache_read_default_ratio: float | None = None,
|
||||
directional_token_threshold: int | None = None,
|
||||
) -> _native.PriceEvaluation | _native.RowErrors:
|
||||
return _native.price_with_rates(
|
||||
dict(row),
|
||||
usage,
|
||||
when,
|
||||
policy="sdk_tokens",
|
||||
project=project,
|
||||
factors=factors,
|
||||
cache_read_default_ratio=cache_read_default_ratio,
|
||||
directional_token_threshold=directional_token_threshold,
|
||||
)
|
||||
|
||||
|
||||
def require_priced(result: _native.Priced | _native.Unpriced | _native.RowErrors, model: str) -> _native.Priced:
|
||||
if isinstance(result, _native.Priced):
|
||||
return result
|
||||
detail: Final = (
|
||||
result.missing
|
||||
if isinstance(result, _native.Unpriced)
|
||||
else tuple(f"{error.field}: {error.message}" for error in result.errors)
|
||||
)
|
||||
raise ModelNotMappedError(f"No valid pricing for model={model}: {detail}")
|
||||
|
||||
|
||||
def billing_context(
|
||||
current_time: datetime,
|
||||
service_tier: str | None = None,
|
||||
channel: Literal["online", "batch", "decisions"] = "online",
|
||||
) -> _native.WhenInput:
|
||||
normalized: Final = service_tier.strip().lower() if service_tier else "standard"
|
||||
tier: Final = (
|
||||
"priority"
|
||||
if normalized == "fast"
|
||||
else normalized
|
||||
if normalized in ("priority", "flex", "ultrafast", "balanced")
|
||||
else "standard"
|
||||
)
|
||||
utc_time: Final = current_time if current_time.tzinfo is not None else current_time.replace(tzinfo=timezone.utc)
|
||||
return _native.WhenInput(int(utc_time.timestamp()), tier=tier, channel=channel)
|
||||
|
||||
|
||||
def cache_read_meter(usage: Usage) -> Literal["cache_read_input_token_cost", "explicit_cache_read_input_token_cost"]:
|
||||
return (
|
||||
"explicit_cache_read_input_token_cost"
|
||||
if getattr(usage.prompt_tokens_details, "cache_type", None) == "ephemeral"
|
||||
else "cache_read_input_token_cost"
|
||||
)
|
||||
|
||||
|
||||
def cache_write_quantities(cache_creation: int, ttl: CacheCreationTokenDetails | None) -> Mapping[str, int]:
|
||||
short_writes: Final = ((ttl.ephemeral_5m_input_tokens or 0) + (ttl.ephemeral_30m_input_tokens or 0)) if ttl else 0
|
||||
writes: Final = {
|
||||
"cache_creation_input_token_cost_above_1hr": (ttl.ephemeral_1h_input_tokens or 0) if ttl else 0,
|
||||
"cache_creation_input_token_cost_above_12hr": (ttl.ephemeral_12h_input_tokens or 0) if ttl else 0,
|
||||
}
|
||||
return {
|
||||
"cache_creation_input_token_cost": max(cache_creation - short_writes - sum(writes.values()), 0) + short_writes,
|
||||
**writes,
|
||||
}
|
||||
|
||||
|
||||
def _prompt_quantities(usage: Usage, row: Mapping[str, object]) -> Mapping[str, int]:
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
|
||||
|
||||
if usage.prompt_tokens_details is None:
|
||||
return {"input_cost_per_token": usage.prompt_tokens}
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cache_hit: Final = details["cache_hit_tokens"]
|
||||
cache_creation: Final = details["cache_creation_tokens"]
|
||||
total_details: Final = sum(
|
||||
(
|
||||
details["text_tokens"],
|
||||
cache_hit,
|
||||
cache_creation,
|
||||
details["audio_tokens"],
|
||||
details["image_tokens"],
|
||||
details["video_tokens"],
|
||||
)
|
||||
)
|
||||
overlaps: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens
|
||||
uncached: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0)
|
||||
audio: Final = min(details["audio_tokens"], uncached) if overlaps else details["audio_tokens"]
|
||||
image: Final = min(details["image_tokens"], uncached - audio) if overlaps else details["image_tokens"]
|
||||
video: Final = min(details["video_tokens"], uncached - audio - image) if overlaps else details["video_tokens"]
|
||||
remainder: Final = max(uncached - audio - image - video, 0)
|
||||
text: Final = (
|
||||
remainder
|
||||
if overlaps or (details["text_tokens"] == 0 and details["image_count"] == 0)
|
||||
else details["text_tokens"]
|
||||
)
|
||||
return {
|
||||
"input_cost_per_token": 0
|
||||
if details["character_count"] and row.get("input_cost_per_character") is not None
|
||||
else text,
|
||||
"input_cost_per_audio_token": 0
|
||||
if details["audio_length_seconds"] and row.get("input_cost_per_audio_per_second") is not None
|
||||
else audio,
|
||||
"input_cost_per_image_token": 0
|
||||
if details["image_count"] and row.get("input_cost_per_image") is not None
|
||||
else image,
|
||||
"input_cost_per_video_token": 0
|
||||
if details["video_length_seconds"] and row.get("input_cost_per_video_per_second") is not None
|
||||
else video,
|
||||
cache_read_meter(usage): cache_hit - details["cache_hit_audio_tokens"],
|
||||
"cache_read_input_audio_token_cost": details["cache_hit_audio_tokens"],
|
||||
**cache_write_quantities(cache_creation, details["cache_creation_token_details"]),
|
||||
}
|
||||
|
||||
|
||||
def _completion_quantities(usage: Usage, *, batch: bool = False) -> Mapping[str, int]:
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_completion_tokens_details
|
||||
|
||||
if usage.completion_tokens_details is None:
|
||||
return {"output_cost_per_token": usage.completion_tokens}
|
||||
details: Final = parse_completion_tokens_details(usage)
|
||||
image_tokens: Final = min(details["image_tokens"], usage.completion_tokens) if batch else details["image_tokens"]
|
||||
modalities: Final = {
|
||||
"output_cost_per_audio_token": details["audio_tokens"],
|
||||
"output_cost_per_reasoning_token": details["reasoning_tokens"],
|
||||
"output_cost_per_image_token": image_tokens,
|
||||
"output_cost_per_video_token": details["video_tokens"],
|
||||
}
|
||||
text: Final = details["text_tokens"] or max(usage.completion_tokens - sum(modalities.values()), 0)
|
||||
return {"output_cost_per_token": text, **modalities}
|
||||
|
||||
|
||||
def token_usage(
|
||||
usage: Usage,
|
||||
row: Mapping[str, object],
|
||||
*,
|
||||
inclusive_threshold: bool = False,
|
||||
batch: bool = False,
|
||||
direction: Literal["input", "output"] | None = None,
|
||||
) -> _native.UsageInput:
|
||||
return _native.UsageInput(
|
||||
{
|
||||
**(_prompt_quantities(usage, row) if direction != "output" else {}),
|
||||
**(_completion_quantities(usage, batch=batch) if direction != "input" else {}),
|
||||
},
|
||||
prompt_tokens=usage.prompt_tokens,
|
||||
inclusive_threshold=inclusive_threshold,
|
||||
)
|
||||
|
|
@ -16,6 +16,8 @@ from typing_extensions import ReadOnly, TypedDict, assert_never
|
|||
import litellm
|
||||
import litellm._logging
|
||||
from litellm import verbose_logger
|
||||
from litellm._cost import request_model_info
|
||||
from litellm._internal_context import current_billing_time
|
||||
from litellm.constants import (
|
||||
DEFAULT_MAX_LRU_CACHE_SIZE,
|
||||
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND,
|
||||
|
|
@ -37,17 +39,11 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
_SERVICE_TIER_TO_COST_KEY_SUFFIX,
|
||||
BilledTokenRates,
|
||||
CostCalculatorUtils,
|
||||
calculate_cache_writing_cost,
|
||||
calculate_cost_component,
|
||||
generic_cost_per_character,
|
||||
generic_cost_per_token,
|
||||
get_batch_cost_rates,
|
||||
get_billable_input_tokens,
|
||||
get_regional_uplift_multiplier,
|
||||
get_service_tier_cost_key,
|
||||
get_token_type_cost_breakdown,
|
||||
get_vertex_regional_endpoint_uplift,
|
||||
parse_prompt_tokens_details,
|
||||
select_cost_metric_for_model,
|
||||
)
|
||||
from litellm.llms.anthropic.cost_calculation import (
|
||||
|
|
@ -239,30 +235,26 @@ def _cost_per_token_custom_pricing_helper(
|
|||
return None
|
||||
|
||||
if custom_cost_per_token is not None:
|
||||
input_cost_per_token: Final = custom_cost_per_token["input_cost_per_token"]
|
||||
output_cost_per_token: Final = custom_cost_per_token["output_cost_per_token"]
|
||||
from litellm._cost import billing_context, price, require_priced
|
||||
from litellm.rust_bridge import _native
|
||||
|
||||
cache_read_input_token_cost: Final = custom_cost_per_token.get(
|
||||
"cache_read_input_token_cost",
|
||||
input_cost_per_token,
|
||||
priced: Final = require_priced(
|
||||
price(
|
||||
{"litellm_provider": "custom", **custom_cost_per_token},
|
||||
_native.UsageInput(
|
||||
{
|
||||
"input_cost_per_token": int(max(prompt_tokens - cached_tokens - cache_creation_tokens, 0)),
|
||||
"cache_read_input_token_cost": int(cached_tokens),
|
||||
"cache_creation_input_token_cost": int(cache_creation_tokens),
|
||||
"output_cost_per_token": int(completion_tokens),
|
||||
},
|
||||
prompt_tokens=int(prompt_tokens),
|
||||
),
|
||||
billing_context(current_billing_time()),
|
||||
),
|
||||
"custom pricing",
|
||||
)
|
||||
cache_creation_input_token_cost: Final = custom_cost_per_token.get(
|
||||
"cache_creation_input_token_cost",
|
||||
input_cost_per_token,
|
||||
)
|
||||
|
||||
regular_prompt_tokens: Final = max(
|
||||
prompt_tokens - cached_tokens - cache_creation_tokens,
|
||||
0,
|
||||
)
|
||||
|
||||
input_cost: Final = (
|
||||
regular_prompt_tokens * input_cost_per_token
|
||||
+ cached_tokens * cache_read_input_token_cost
|
||||
+ cache_creation_tokens * cache_creation_input_token_cost
|
||||
)
|
||||
output_cost = completion_tokens * output_cost_per_token
|
||||
return input_cost, output_cost
|
||||
return priced.input, priced.output
|
||||
elif custom_cost_per_second is not None:
|
||||
output_cost = custom_cost_per_second * (response_time_ms or 0.0) / 1000
|
||||
return 0, output_cost
|
||||
|
|
@ -369,9 +361,9 @@ def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool:
|
|||
return mode is None or mode in _WALL_CLOCK_PRICED_MODES
|
||||
|
||||
|
||||
def _model_info_for_pricing(model: str, custom_llm_provider: str | None) -> ModelInfoBase | None:
|
||||
def _model_info_for_pricing(model: str, custom_llm_provider: str | None) -> ModelInfo | None:
|
||||
try:
|
||||
return cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
|
||||
return None
|
||||
|
||||
|
|
@ -807,18 +799,16 @@ def cost_per_token(
|
|||
model_info: Final = (
|
||||
pricing_model_info
|
||||
if pricing_model_info is not None
|
||||
else cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
else litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
)
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
model_info=model_info,
|
||||
)
|
||||
if _has_token_or_tiered_pricing(model_info):
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
verbose_logger.debug("No per-token, tiered, or per-second pricing for model=%s; cost is 0", model)
|
||||
return 0.0, 0.0
|
||||
|
||||
|
||||
def get_replicate_completion_pricing(completion_response: dict, total_time=0.0):
|
||||
|
|
@ -909,7 +899,12 @@ def _bedrock_pricing_info(
|
|||
)
|
||||
profile: Final = _cost_map_model_info(invoked_model_id, "bedrock") if invoked_model_id else None
|
||||
if profile is None or not _declares_rates(profile):
|
||||
return _cost_map_model_info(registration[0], None) if registration is not None else None
|
||||
if registration is None:
|
||||
return None
|
||||
source: Final = _raw_cost_map_entry(registration[0])
|
||||
return _cost_map_model_info(
|
||||
registration[0], "bedrock" if source is not None and "litellm_provider" not in source else None
|
||||
)
|
||||
if registration is None or registration[0] == profile["key"]:
|
||||
return profile
|
||||
return model_info_with_pricing_overrides(profile, registration[1])
|
||||
|
|
@ -1512,6 +1507,35 @@ def _response_time_ms_for_cost(
|
|||
return 0.0
|
||||
|
||||
|
||||
def _request_pricing_cost(
|
||||
*,
|
||||
model: str,
|
||||
provider: str | None,
|
||||
prices: Mapping[str, object] | None,
|
||||
usage: Usage,
|
||||
response_time_ms: float | None,
|
||||
audio_seconds: float,
|
||||
service_tier: str | None,
|
||||
data_residency: str | None,
|
||||
vertex_location: str | None,
|
||||
) -> tuple[ModelInfo | None, tuple[float, float] | None]:
|
||||
if prices is None:
|
||||
return None, None
|
||||
info: Final = request_model_info(model, provider, prices)
|
||||
time_cost: Final = _per_second_pricing_cost(model, info, response_time_ms, audio_seconds)
|
||||
if time_cost is not None:
|
||||
return info, time_cost
|
||||
return info, generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider=provider or "custom",
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
model_info=info,
|
||||
)
|
||||
|
||||
|
||||
def completion_cost(
|
||||
completion_response: object | None = None,
|
||||
model: str | None = None,
|
||||
|
|
@ -2032,36 +2056,54 @@ def completion_cost(
|
|||
if litellm_logging_obj is not None:
|
||||
request_model_for_cost = litellm_logging_obj.model
|
||||
|
||||
(
|
||||
prompt_tokens_cost_usd_dollar,
|
||||
completion_tokens_cost_usd_dollar,
|
||||
) = cost_per_token(
|
||||
request_info, request_cost = _request_pricing_cost(
|
||||
model=model,
|
||||
prompt_tokens=prompt_tokens or 0,
|
||||
completion_tokens=completion_tokens or 0,
|
||||
custom_llm_provider=_pricing_provider_for_cost_candidate(
|
||||
candidate_provider=candidate_pricing_provider,
|
||||
request_provider=request_pricing_provider,
|
||||
current_provider=custom_llm_provider,
|
||||
),
|
||||
provider=custom_llm_provider,
|
||||
prices=litellm_logging_obj.request_pricing
|
||||
if custom_pricing and litellm_logging_obj is not None and custom_cost_per_token is None
|
||||
else None,
|
||||
usage=cost_per_token_usage_object
|
||||
or Usage(prompt_tokens=prompt_tokens or 0, completion_tokens=completion_tokens or 0),
|
||||
response_time_ms=response_time_ms,
|
||||
region_name=None if explicit_pricing else region_name,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
prompt_characters=prompt_characters,
|
||||
completion_characters=completion_characters,
|
||||
cache_creation_input_tokens=cache_creation_input_tokens,
|
||||
cache_read_input_tokens=cache_read_input_tokens,
|
||||
usage_object=cost_per_token_usage_object,
|
||||
call_type=call_type,
|
||||
audio_transcription_file_duration=audio_transcription_file_duration,
|
||||
rerank_billed_units=rerank_billed_units,
|
||||
audio_seconds=audio_transcription_file_duration or 0.0,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
response=completion_response,
|
||||
custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id),
|
||||
bedrock_model_info=bedrock_model_info,
|
||||
)
|
||||
(
|
||||
prompt_tokens_cost_usd_dollar,
|
||||
completion_tokens_cost_usd_dollar,
|
||||
) = (
|
||||
request_cost
|
||||
if request_cost is not None
|
||||
else cost_per_token(
|
||||
model=model,
|
||||
prompt_tokens=prompt_tokens or 0,
|
||||
completion_tokens=completion_tokens or 0,
|
||||
custom_llm_provider=_pricing_provider_for_cost_candidate(
|
||||
candidate_provider=candidate_pricing_provider,
|
||||
request_provider=request_pricing_provider,
|
||||
current_provider=custom_llm_provider,
|
||||
),
|
||||
response_time_ms=response_time_ms,
|
||||
region_name=None if explicit_pricing else region_name,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
prompt_characters=prompt_characters,
|
||||
completion_characters=completion_characters,
|
||||
cache_creation_input_tokens=cache_creation_input_tokens,
|
||||
cache_read_input_tokens=cache_read_input_tokens,
|
||||
usage_object=cost_per_token_usage_object,
|
||||
call_type=call_type,
|
||||
audio_transcription_file_duration=audio_transcription_file_duration,
|
||||
rerank_billed_units=rerank_billed_units,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
response=completion_response,
|
||||
custom_model_info=_ocr_model_info(litellm_logging_obj, custom_pricing, router_model_id),
|
||||
bedrock_model_info=bedrock_model_info,
|
||||
)
|
||||
)
|
||||
|
||||
# Get additional costs from provider (e.g., routing fees, infrastructure costs)
|
||||
|
|
@ -2153,7 +2195,7 @@ def completion_cost(
|
|||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
model_info=bedrock_model_info,
|
||||
model_info=request_info if request_info is not None else bedrock_model_info,
|
||||
)
|
||||
_reasoning_cost = _token_type_breakdown.reasoning_cost
|
||||
_cache_read_cost = _token_type_breakdown.cache_read_cost
|
||||
|
|
@ -2889,32 +2931,26 @@ def default_video_cost_calculator(
|
|||
return 0.0
|
||||
|
||||
|
||||
def _batch_rate(
|
||||
model_info: ModelInfo,
|
||||
key: Literal[
|
||||
"input_cost_per_audio_token_batches",
|
||||
"cache_read_input_audio_token_cost_batches",
|
||||
"input_cost_per_image_token_batches",
|
||||
"input_cost_per_video_token_batches",
|
||||
],
|
||||
fallback: float,
|
||||
) -> float:
|
||||
rate: Final = model_info.get(key)
|
||||
return fallback if rate is None else rate
|
||||
|
||||
|
||||
def _batch_or_half(batch_rate: float | None, standard_rate: float | None, default: float = 0.0) -> float:
|
||||
if batch_rate is not None:
|
||||
return batch_rate
|
||||
if standard_rate is not None:
|
||||
return standard_rate / 2
|
||||
return default
|
||||
|
||||
|
||||
def _completion_image_tokens(usage: Usage) -> int:
|
||||
details: Final = usage.completion_tokens_details
|
||||
image_tokens: Final = (details.image_tokens or 0) if details is not None else 0
|
||||
return min(image_tokens, usage.completion_tokens)
|
||||
def _batch_model_info(model: str, provider: str | None, model_info: ModelInfo | None) -> ModelInfo | None:
|
||||
if model_info is not None and any(
|
||||
model_info.get(key) is not None
|
||||
for key in (
|
||||
"input_cost_per_token_batches",
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token_batches",
|
||||
"output_cost_per_token",
|
||||
)
|
||||
):
|
||||
return model_info
|
||||
try:
|
||||
global_info: Final = litellm.get_model_info(model=model, custom_llm_provider=provider)
|
||||
except Exception:
|
||||
return model_info
|
||||
if model_info is None:
|
||||
return global_info
|
||||
return cast( # cast-ok: both public ModelInfo rows keep their validated field types on merge
|
||||
ModelInfo, {**global_info, **{key: value for key, value in model_info.items() if value is not None}}
|
||||
)
|
||||
|
||||
|
||||
def batch_cost_calculator(
|
||||
|
|
@ -2926,143 +2962,28 @@ def batch_cost_calculator(
|
|||
*,
|
||||
vertex_location: str | None = None,
|
||||
) -> tuple[float, float]:
|
||||
"""
|
||||
Calculate the cost of a batch job.
|
||||
from litellm._cost import billing_context, price, require_priced, token_usage
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import uses_inclusive_token_thresholds
|
||||
|
||||
Args:
|
||||
model_info: Optional deployment-level model info containing custom
|
||||
batch pricing (e.g. input_cost_per_token_batches). When provided,
|
||||
skips the global litellm.get_model_info() lookup so that
|
||||
deployment-specific pricing is used.
|
||||
"""
|
||||
|
||||
_, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
verbose_logger.debug(
|
||||
"Calculating batch cost per token. model=%s, custom_llm_provider=%s",
|
||||
model,
|
||||
custom_llm_provider,
|
||||
)
|
||||
|
||||
if model_info is None:
|
||||
try:
|
||||
model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception:
|
||||
model_info = None
|
||||
elif not any(
|
||||
model_info.get(k) is not None
|
||||
for k in (
|
||||
"input_cost_per_token_batches",
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token_batches",
|
||||
"output_cost_per_token",
|
||||
)
|
||||
):
|
||||
deployment_info: Final = model_info
|
||||
try:
|
||||
global_info: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
if global_info:
|
||||
model_info = {
|
||||
**global_info,
|
||||
**{key: value for key, value in deployment_info.items() if value is not None},
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not model_info:
|
||||
return 0.0, 0.0
|
||||
|
||||
batch_rates: Final = get_batch_cost_rates(model_info, usage, custom_llm_provider)
|
||||
input_cost_per_token: Final = model_info.get("input_cost_per_token")
|
||||
output_cost_per_token: Final = model_info.get("output_cost_per_token")
|
||||
total_prompt_cost = 0.0
|
||||
total_completion_cost = 0.0
|
||||
if batch_rates.input is not None:
|
||||
total_prompt_cost = _batch_prompt_cost(
|
||||
usage,
|
||||
model_info,
|
||||
batch_rates.input,
|
||||
batch_rates.cache_read,
|
||||
batch_rates.cache_creation,
|
||||
batch_rates.cache_creation_rates_by_ttl,
|
||||
)
|
||||
elif input_cost_per_token:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cache_read_tokens: Final = details["cache_hit_tokens"]
|
||||
cache_creation_tokens: Final = details["cache_creation_tokens"]
|
||||
|
||||
# Subtract cached tokens from prompt_tokens before calculating cost
|
||||
# Fixes issue where cached tokens are being charged again
|
||||
base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens
|
||||
total_prompt_cost = (
|
||||
base_input_tokens * (input_cost_per_token) / 2
|
||||
) # batch cost is usually half of the regular token cost
|
||||
|
||||
# Add cache read cost if applicable
|
||||
cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None)
|
||||
total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2
|
||||
|
||||
cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token
|
||||
total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
|
||||
text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token)
|
||||
image_rate: Final = _batch_or_half(
|
||||
model_info.get("output_cost_per_image_token_batches"),
|
||||
model_info.get("output_cost_per_image_token"),
|
||||
default=text_rate,
|
||||
)
|
||||
image_tokens: Final = _completion_image_tokens(usage)
|
||||
text_tokens: Final = usage.completion_tokens - image_tokens
|
||||
total_completion_cost = text_tokens * text_rate + image_tokens * image_rate
|
||||
|
||||
uplift: Final = get_regional_uplift_multiplier(model_info, data_residency)
|
||||
if uplift != 1.0:
|
||||
total_prompt_cost *= uplift
|
||||
total_completion_cost *= uplift
|
||||
|
||||
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
|
||||
return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift
|
||||
|
||||
|
||||
def _batch_prompt_cost(
|
||||
usage: Usage,
|
||||
model_info: ModelInfo,
|
||||
input_rate: float,
|
||||
cache_read_rate: float | None,
|
||||
cache_creation_rate: float | None,
|
||||
cache_creation_rates_by_ttl: Mapping[str, float],
|
||||
) -> float:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
|
||||
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
|
||||
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
|
||||
audio_tokens, image_tokens, video_tokens = (
|
||||
details["audio_tokens"],
|
||||
details["image_tokens"],
|
||||
details["video_tokens"],
|
||||
)
|
||||
text_tokens: Final = max(
|
||||
(usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens - cached_tokens - written_tokens, 0
|
||||
)
|
||||
return (
|
||||
text_tokens * input_rate
|
||||
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
|
||||
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
|
||||
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
|
||||
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
|
||||
+ cached_audio_tokens
|
||||
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
|
||||
+ (
|
||||
calculate_cache_writing_cost(
|
||||
written_tokens,
|
||||
details["cache_creation_token_details"],
|
||||
input_rate if cache_creation_rate is None else cache_creation_rate,
|
||||
cache_creation_rates_by_ttl,
|
||||
_, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
resolved: Final = _batch_model_info(model, provider, model_info)
|
||||
row: Final = dict(resolved or {})
|
||||
pricing_row: Final = row if "litellm_provider" in row else {"litellm_provider": provider or "custom", **row}
|
||||
priced: Final = require_priced(
|
||||
price(
|
||||
pricing_row,
|
||||
token_usage(usage, pricing_row, inclusive_threshold=uses_inclusive_token_thresholds(provider), batch=True),
|
||||
billing_context(current_billing_time(), channel="batch"),
|
||||
factors=(
|
||||
get_regional_uplift_multiplier(resolved, data_residency),
|
||||
get_vertex_regional_endpoint_uplift(resolved, vertex_location),
|
||||
)
|
||||
if bills_cache_writes
|
||||
else 0.0
|
||||
)
|
||||
if resolved is not None
|
||||
else (),
|
||||
),
|
||||
model,
|
||||
)
|
||||
return priced.input, priced.output
|
||||
|
||||
|
||||
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
|
||||
|
|
|
|||
|
|
@ -626,6 +626,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
slack_app
|
||||
custom_pricing: bool = False
|
||||
custom_cost_per_token: CostPerToken | None = None
|
||||
request_pricing: Mapping[str, object] | None = None
|
||||
custom_cost_per_second: float | None = None
|
||||
replaced_global_callback_keys: frozenset[str] = frozenset()
|
||||
stream_options = None
|
||||
|
|
@ -804,6 +805,11 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
|
||||
self.litellm_params = litellm_params
|
||||
|
||||
if kwargs is not None:
|
||||
from litellm._cost import request_prices
|
||||
|
||||
self.request_pricing = request_prices(kwargs)
|
||||
|
||||
# Initialize cost breakdown field
|
||||
self.cost_breakdown: CostBreakdown | None = None
|
||||
self.billed_token_rates: BilledTokenRates | None = None
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -9,15 +9,12 @@ from pydantic import ValidationError
|
|||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
generic_cost_per_token,
|
||||
get_provider_specific_geo_multiplier,
|
||||
get_provider_specific_speed_multiplier,
|
||||
get_web_search_requests_from_usage,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
import litellm
|
||||
|
||||
|
||||
def cost_per_token(
|
||||
|
|
@ -36,7 +33,7 @@ def cost_per_token(
|
|||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="anthropic",
|
||||
|
|
@ -44,28 +41,6 @@ def cost_per_token(
|
|||
model_info=model_info,
|
||||
)
|
||||
|
||||
# Apply provider_specific_entry multipliers for geo/speed routing
|
||||
try:
|
||||
effective_info: Final = (
|
||||
model_info
|
||||
if model_info is not None
|
||||
else litellm.get_model_info(model=model, custom_llm_provider="anthropic")
|
||||
)
|
||||
geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=effective_info, usage=usage)
|
||||
speed_multiplier: Final = get_provider_specific_speed_multiplier(model_info=effective_info, usage=usage)
|
||||
|
||||
if speed_multiplier != 1.0:
|
||||
prompt_cost *= speed_multiplier
|
||||
completion_cost *= speed_multiplier
|
||||
|
||||
if geo_multiplier != 1.0:
|
||||
prompt_cost *= geo_multiplier
|
||||
completion_cost *= geo_multiplier
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
|
||||
|
||||
class _AnthropicServerToolUseProbe(LiteLLMBaseModel):
|
||||
web_search_requests: int | None = None
|
||||
|
|
|
|||
|
|
@ -5,8 +5,11 @@ Handles Azure AI Foundry Model Router flat cost and other Azure AI specific pric
|
|||
|
||||
from typing import Final
|
||||
|
||||
from litellm._cost import billing_context, price, require_priced
|
||||
from litellm._internal_context import current_billing_time
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.rust_bridge import _native
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
|
@ -57,10 +60,14 @@ def calculate_azure_model_router_flat_cost(model: str, prompt_tokens: int) -> fl
|
|||
if not is_azure_model_router(model):
|
||||
return 0.0
|
||||
model_info: Final = get_model_info(model=_router_fee_entry_name(model), custom_llm_provider="azure_ai")
|
||||
router_flat_cost_per_token: Final = model_info.get("input_cost_per_token", 0)
|
||||
if router_flat_cost_per_token and router_flat_cost_per_token > 0:
|
||||
return prompt_tokens * router_flat_cost_per_token
|
||||
return 0.0
|
||||
return require_priced(
|
||||
price(
|
||||
model_info,
|
||||
_native.UsageInput({"input_cost_per_token": prompt_tokens}, prompt_tokens=prompt_tokens),
|
||||
billing_context(current_billing_time()),
|
||||
),
|
||||
model,
|
||||
).input
|
||||
|
||||
|
||||
def _response_model_cost(model: str, usage: Usage, service_tier: str | None) -> tuple[float, float]:
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Final
|
|||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
generic_cost_per_token,
|
||||
get_provider_specific_geo_multiplier,
|
||||
price_token_usage,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -22,12 +22,12 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None)
|
|||
Follows the same logic as Anthropic's cost per token calculation.
|
||||
"""
|
||||
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="bedrock")
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
return price_token_usage(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="bedrock",
|
||||
service_tier=service_tier,
|
||||
model_info=model_info,
|
||||
resolved_model_info=model_info,
|
||||
current_time=None,
|
||||
factors=(get_provider_specific_geo_multiplier(model_info=model_info, usage=usage),),
|
||||
)
|
||||
geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
|
||||
return prompt_cost * geo_multiplier, completion_cost * geo_multiplier
|
||||
|
|
|
|||
|
|
@ -1,113 +1,7 @@
|
|||
"""
|
||||
Cost calculator for Dashscope Chat models.
|
||||
|
||||
Alibaba Model Studio tiered pricing is all-or-nothing: the tier is picked from the
|
||||
total input tokens of a single request, and every token of that request (input,
|
||||
cached, cache-creation, output, reasoning) is billed at that one tier's rate.
|
||||
See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
TokenRates,
|
||||
apply_off_peak_pricing,
|
||||
parse_completion_tokens_details,
|
||||
parse_prompt_tokens_details,
|
||||
resolve_explicit_cache_read_cost_rate,
|
||||
)
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TokenBreakdown:
|
||||
text_tokens: int
|
||||
cached_tokens: int
|
||||
cache_creation_tokens: int
|
||||
completion_tokens: int
|
||||
reasoning_tokens: int
|
||||
|
||||
@property
|
||||
def total_input_tokens(self) -> int:
|
||||
return self.text_tokens + self.cached_tokens + self.cache_creation_tokens
|
||||
|
||||
|
||||
def _extract_token_breakdown(usage: Usage) -> TokenBreakdown:
|
||||
prompt_details: Final = parse_prompt_tokens_details(usage)
|
||||
cached_tokens: Final = prompt_details["cache_hit_tokens"]
|
||||
cache_creation_tokens: Final = prompt_details["cache_creation_tokens"]
|
||||
text_tokens: Final = max(usage.prompt_tokens - cached_tokens - cache_creation_tokens, 0)
|
||||
|
||||
reasoning_tokens: Final = parse_completion_tokens_details(usage)["reasoning_tokens"]
|
||||
completion_tokens: Final = max((usage.completion_tokens or 0) - reasoning_tokens, 0)
|
||||
|
||||
return TokenBreakdown(
|
||||
text_tokens=text_tokens,
|
||||
cached_tokens=cached_tokens,
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
reasoning_tokens=reasoning_tokens,
|
||||
)
|
||||
|
||||
|
||||
def _flat_rate(value: float | None, fallback_value: float | None) -> float:
|
||||
if value is None:
|
||||
return float(fallback_value or 0.0)
|
||||
return float(value)
|
||||
|
||||
|
||||
def _flat_rates(model_info: ModelInfo) -> TokenRates:
|
||||
reasoning_rate: Final = model_info.get("output_cost_per_reasoning_token")
|
||||
return TokenRates(
|
||||
input_rate=float(model_info.get("input_cost_per_token") or 0.0),
|
||||
cache_read_rate=_flat_rate(
|
||||
model_info.get("cache_read_input_token_cost"), model_info.get("input_cost_per_token")
|
||||
),
|
||||
cache_creation_rate=_flat_rate(
|
||||
model_info.get("cache_creation_input_token_cost"), model_info.get("input_cost_per_token")
|
||||
),
|
||||
output_rate=float(model_info.get("output_cost_per_token") or 0.0),
|
||||
reasoning_rate=None if reasoning_rate is None else float(reasoning_rate),
|
||||
)
|
||||
|
||||
|
||||
def _tier_rates(model_info: ModelInfo, tier: Mapping[str, object]) -> TokenRates:
|
||||
# A tier that declares output rates keeps the request on them, all-or-nothing. A tier table
|
||||
# spelling out only input rates would serve every completion for free, so there the model's
|
||||
# own output rates stand in
|
||||
flat_rates: Final = _flat_rates(model_info)
|
||||
tier_declares_output: Final = "output_cost_per_token" in tier
|
||||
tier_declares_reasoning: Final = "output_cost_per_reasoning_token" in tier
|
||||
return TokenRates(
|
||||
input_rate=tier_rate(tier, "input_cost_per_token"),
|
||||
cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
|
||||
cache_creation_rate=tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token"),
|
||||
output_rate=tier_rate(tier, "output_cost_per_token") if tier_declares_output else flat_rates.output_rate,
|
||||
reasoning_rate=(
|
||||
tier_rate(tier, "output_cost_per_reasoning_token")
|
||||
if tier_declares_reasoning
|
||||
else None
|
||||
if tier_declares_output
|
||||
else flat_rates.reasoning_rate
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _bill(breakdown: TokenBreakdown, rates: TokenRates) -> tuple[float, float]:
|
||||
prompt_cost: Final = (
|
||||
(breakdown.text_tokens * rates.input_rate)
|
||||
+ (breakdown.cached_tokens * rates.cache_read_rate)
|
||||
+ (breakdown.cache_creation_tokens * rates.cache_creation_rate)
|
||||
)
|
||||
completion_cost: Final = (breakdown.completion_tokens * rates.output_rate) + (
|
||||
breakdown.reasoning_tokens * rates.billed_reasoning_rate
|
||||
)
|
||||
return prompt_cost, completion_cost
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
def cost_per_token(
|
||||
|
|
@ -116,39 +10,9 @@ def cost_per_token(
|
|||
custom_llm_provider: str = "dashscope",
|
||||
current_time: datetime | None = None,
|
||||
) -> tuple[float, float]:
|
||||
"""
|
||||
Calculate cost per token for Dashscope models.
|
||||
|
||||
Supports both tiered and flat pricing with cached and reasoning tokens, and swaps in the
|
||||
model's off_peak_pricing rates while one of its windows is open.
|
||||
|
||||
Args:
|
||||
model: Model name without provider prefix
|
||||
usage: LiteLLM Usage block
|
||||
custom_llm_provider: The provider id the request resolved to; dashscope or one of its brand aliases
|
||||
current_time: The moment the request is billed at; defaults to now, UTC
|
||||
|
||||
Returns:
|
||||
Tuple[float, float] - (prompt_cost_in_usd, completion_cost_in_usd)
|
||||
"""
|
||||
model_info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
breakdown: Final = _extract_token_breakdown(usage)
|
||||
raw_tiers: Final = model_info.get("tiered_pricing")
|
||||
tiered_pricing: Final = raw_tiers if isinstance(raw_tiers, list) else None
|
||||
tier: Final = (
|
||||
select_tier_for_input(tiered_pricing=tiered_pricing, input_tokens=breakdown.total_input_tokens)
|
||||
if tiered_pricing
|
||||
else None
|
||||
)
|
||||
standard_rates: Final = _flat_rates(model_info) if tier is None else _tier_rates(model_info, tier)
|
||||
rates: Final = apply_off_peak_pricing(model_info, current_time, standard_rates)
|
||||
explicit_cache_read_rate: Final = resolve_explicit_cache_read_cost_rate(
|
||||
model_info=model_info,
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
current_time=current_time,
|
||||
)
|
||||
billed_rates: Final = (
|
||||
rates if explicit_cache_read_rate is None else replace(rates, cache_read_rate=explicit_cache_read_rate)
|
||||
)
|
||||
|
||||
return _bill(breakdown, billed_rates)
|
||||
|
|
|
|||
|
|
@ -1,42 +0,0 @@
|
|||
from typing import (
|
||||
Final,
|
||||
cast, # noqa: TID251 # the derived entry is a dict copy of a ReadOnly TypedDict; no cast-free way to retype it
|
||||
)
|
||||
|
||||
from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
|
||||
from litellm.types.utils import ModelInfo
|
||||
|
||||
|
||||
def _as_rate(value: object) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def with_default_cache_read_rate(model_info: ModelInfo) -> ModelInfo:
|
||||
input_rate: Final = _as_rate(model_info.get("input_cost_per_token"))
|
||||
if model_info.get("cache_read_input_token_cost") is not None or input_rate is None:
|
||||
return model_info
|
||||
cache_read_rate: Final = input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
|
||||
off_peak: Final = model_info.get("off_peak_pricing")
|
||||
if off_peak is None or "cache_read_input_token_cost" in off_peak:
|
||||
return cast(ModelInfo, {**model_info, "cache_read_input_token_cost": cache_read_rate})
|
||||
off_peak_input_rate: Final = _as_rate(off_peak.get("input_cost_per_token"))
|
||||
return cast(
|
||||
ModelInfo,
|
||||
{
|
||||
**model_info,
|
||||
"cache_read_input_token_cost": cache_read_rate,
|
||||
"off_peak_pricing": {
|
||||
**off_peak,
|
||||
"cache_read_input_token_cost": (
|
||||
off_peak_input_rate * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
|
||||
if off_peak_input_rate is not None
|
||||
else cache_read_rate
|
||||
),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
|
@ -6,7 +6,9 @@ Helper util for handling perplexity-specific cost calculation
|
|||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing
|
||||
from litellm._cost import billing_context, price, require_priced
|
||||
from litellm._internal_context import current_billing_time
|
||||
from litellm.rust_bridge import _native
|
||||
from litellm.types.utils import Usage
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
|
@ -52,22 +54,31 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
|
|||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
rates: Final = apply_off_peak_pricing(
|
||||
model_info,
|
||||
current_time,
|
||||
TokenRates(
|
||||
input_rate=_safe_float_cast(model_info.get("input_cost_per_token")),
|
||||
output_rate=_safe_float_cast(model_info.get("output_cost_per_token")),
|
||||
cache_read_rate=0.0,
|
||||
cache_creation_rate=0.0,
|
||||
reasoning_rate=None,
|
||||
),
|
||||
top_reasoning: Final[object] = getattr(usage, "reasoning_tokens", None)
|
||||
reasoning_tokens: Final = (
|
||||
top_reasoning
|
||||
if isinstance(top_reasoning, int) and top_reasoning > 0
|
||||
else (usage.completion_tokens_details.reasoning_tokens or 0)
|
||||
if usage.completion_tokens_details is not None
|
||||
else 0
|
||||
)
|
||||
input_cost_per_token: Final = rates.input_rate
|
||||
output_cost_per_token: Final = rates.output_rate
|
||||
|
||||
## CALCULATE INPUT COST
|
||||
prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token
|
||||
priced: Final = require_priced(
|
||||
price(
|
||||
{"litellm_provider": "perplexity", **model_info},
|
||||
_native.UsageInput(
|
||||
{
|
||||
"input_cost_per_token": usage.prompt_tokens,
|
||||
"output_cost_per_token": max(usage.completion_tokens - reasoning_tokens, 0),
|
||||
"output_cost_per_reasoning_token": reasoning_tokens,
|
||||
},
|
||||
prompt_tokens=usage.prompt_tokens,
|
||||
),
|
||||
billing_context(current_time if current_time is not None else current_billing_time()),
|
||||
),
|
||||
model,
|
||||
)
|
||||
prompt_cost: float = priced.input
|
||||
completion_cost: float = priced.output
|
||||
|
||||
## ADD CITATION TOKENS COST (if present)
|
||||
citation_tokens: Final = getattr(usage, "citation_tokens", 0) or 0
|
||||
|
|
@ -76,24 +87,6 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
|
|||
citation_cost_per_token: Final = _safe_float_cast(citation_cost_value)
|
||||
prompt_cost += citation_tokens * citation_cost_per_token
|
||||
|
||||
## CALCULATE OUTPUT COST
|
||||
reasoning_tokens = getattr(usage, "reasoning_tokens", 0) or 0
|
||||
if reasoning_tokens == 0 and hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details:
|
||||
reasoning_tokens = getattr(usage.completion_tokens_details, "reasoning_tokens", 0) or 0
|
||||
|
||||
reasoning_cost_value: Final = model_info.get("output_cost_per_reasoning_token")
|
||||
|
||||
# `completion_tokens` includes `reasoning_tokens` per the OpenAI/Perplexity usage
|
||||
# convention (codified for the central path in PR #18607). When a reasoning rate is
|
||||
# configured we subtract before the output-rate multiplication so the reasoning
|
||||
# tokens are not billed twice.
|
||||
if reasoning_tokens > 0 and reasoning_cost_value is not None:
|
||||
non_reasoning_completion_tokens: Final = max(0, (usage.completion_tokens or 0) - reasoning_tokens)
|
||||
completion_cost: float = non_reasoning_completion_tokens * output_cost_per_token
|
||||
completion_cost += reasoning_tokens * _safe_float_cast(reasoning_cost_value)
|
||||
else:
|
||||
completion_cost = (usage.completion_tokens or 0) * output_cost_per_token
|
||||
|
||||
## ADD SEARCH QUERIES COST (if present)
|
||||
num_search_queries = 0
|
||||
if hasattr(usage, "prompt_tokens_details") and usage.prompt_tokens_details:
|
||||
|
|
|
|||
|
|
@ -1,14 +1,18 @@
|
|||
# What is this?
|
||||
## Cost calculation for Google AI Studio / Vertex AI models
|
||||
from collections.abc import Sequence
|
||||
from typing import Final, Literal
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm._cost import billing_context, price, require_priced, token_usage
|
||||
from litellm._internal_context import current_billing_time
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
generic_cost_per_token,
|
||||
get_vertex_regional_endpoint_uplift,
|
||||
is_above_128k,
|
||||
)
|
||||
from litellm.rust_bridge import _native
|
||||
from litellm.types.utils import ModelInfo, Usage
|
||||
|
||||
"""
|
||||
|
|
@ -58,6 +62,30 @@ def cost_router(
|
|||
return "cost_per_character"
|
||||
|
||||
|
||||
def _token_component_cost(
|
||||
model: str,
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
direction: Literal["input", "output"],
|
||||
service_tier: str | None,
|
||||
) -> float:
|
||||
if (
|
||||
model_info.get("input_cost_per_token_above_128k_tokens") is not None
|
||||
or model_info.get("output_cost_per_token_above_128k_tokens") is not None
|
||||
):
|
||||
input_cost, output_cost = _handle_128k_pricing(model_info, usage, direction=direction)
|
||||
return input_cost if direction == "input" else output_cost
|
||||
priced: Final = require_priced(
|
||||
price(
|
||||
model_info,
|
||||
token_usage(usage, model_info, direction=direction),
|
||||
billing_context(current_billing_time(), service_tier),
|
||||
),
|
||||
model,
|
||||
)
|
||||
return priced.input if direction == "input" else priced.output
|
||||
|
||||
|
||||
def cost_per_character(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
|
|
@ -91,12 +119,7 @@ def cost_per_character(
|
|||
|
||||
## CALCULATE INPUT COST
|
||||
if prompt_characters is None:
|
||||
prompt_cost, _ = cost_per_token(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
prompt_cost = _token_component_cost(model, model_info, usage, "input", service_tier)
|
||||
else:
|
||||
try:
|
||||
if (
|
||||
|
|
@ -123,21 +146,11 @@ def cost_per_character(
|
|||
"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - %s\nDefaulting to None",
|
||||
e,
|
||||
)
|
||||
prompt_cost, _ = cost_per_token(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
prompt_cost = _token_component_cost(model, model_info, usage, "input", service_tier)
|
||||
|
||||
## CALCULATE OUTPUT COST
|
||||
if completion_characters is None:
|
||||
_, completion_cost = cost_per_token(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
completion_cost = _token_component_cost(model, model_info, usage, "output", service_tier)
|
||||
else:
|
||||
completion_tokens: Final = usage.completion_tokens
|
||||
try:
|
||||
|
|
@ -164,12 +177,7 @@ def cost_per_character(
|
|||
"litellm.litellm_core_utils.llm_cost_calc.google.py::cost_per_character(): Exception occured - %s\nDefaulting to None",
|
||||
e,
|
||||
)
|
||||
_, completion_cost = cost_per_token(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
completion_cost = _token_component_cost(model, model_info, usage, "output", service_tier)
|
||||
|
||||
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
|
||||
return prompt_cost * vertex_uplift, completion_cost * vertex_uplift
|
||||
|
|
@ -178,27 +186,27 @@ def cost_per_character(
|
|||
def _handle_128k_pricing(
|
||||
model_info: ModelInfo,
|
||||
usage: Usage,
|
||||
*,
|
||||
factors: Sequence[float] = (),
|
||||
direction: Literal["input", "output"] | None = None,
|
||||
) -> tuple[float, float]:
|
||||
## CALCULATE INPUT COST
|
||||
input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens")
|
||||
output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens")
|
||||
|
||||
prompt_tokens: Final = usage.prompt_tokens
|
||||
completion_tokens: Final = usage.completion_tokens
|
||||
|
||||
if is_above_128k(tokens=prompt_tokens) and input_cost_per_token_above_128k_tokens is not None:
|
||||
prompt_cost = prompt_tokens * input_cost_per_token_above_128k_tokens
|
||||
else:
|
||||
prompt_cost = prompt_tokens * (model_info["input_cost_per_token"] or 0.0)
|
||||
|
||||
## CALCULATE OUTPUT COST
|
||||
output_cost_per_token_above_128k_tokens = model_info.get("output_cost_per_token_above_128k_tokens")
|
||||
if is_above_128k(tokens=completion_tokens) and output_cost_per_token_above_128k_tokens is not None:
|
||||
completion_cost = completion_tokens * output_cost_per_token_above_128k_tokens
|
||||
else:
|
||||
completion_cost = completion_tokens * (model_info["output_cost_per_token"] or 0.0)
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
result: Final = require_priced(
|
||||
price(
|
||||
model_info,
|
||||
_native.UsageInput(
|
||||
{
|
||||
**({"input_cost_per_token": usage.prompt_tokens} if direction != "output" else {}),
|
||||
**({"output_cost_per_token": usage.completion_tokens} if direction != "input" else {}),
|
||||
},
|
||||
prompt_tokens=usage.prompt_tokens,
|
||||
),
|
||||
billing_context(current_billing_time()),
|
||||
factors=factors,
|
||||
directional_token_threshold=128000,
|
||||
),
|
||||
model_info.get("key", "vertex_ai"),
|
||||
)
|
||||
return result.input, result.output
|
||||
|
||||
|
||||
def cost_per_token(
|
||||
|
|
@ -235,12 +243,11 @@ def cost_per_token(
|
|||
input_cost_per_token_above_128k_tokens: Final = model_info.get("input_cost_per_token_above_128k_tokens")
|
||||
output_cost_per_token_above_128k_tokens: Final = model_info.get("output_cost_per_token_above_128k_tokens")
|
||||
if input_cost_per_token_above_128k_tokens is not None or output_cost_per_token_above_128k_tokens is not None:
|
||||
prompt_cost_128k, completion_cost_128k = _handle_128k_pricing(
|
||||
return _handle_128k_pricing(
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
factors=(get_vertex_regional_endpoint_uplift(model_info, vertex_location),),
|
||||
)
|
||||
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
|
||||
return prompt_cost_128k * vertex_uplift, completion_cost_128k * vertex_uplift
|
||||
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
|
|
|
|||
122
litellm/main.py
122
litellm/main.py
|
|
@ -155,7 +155,6 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import (
|
||||
BaseUrlRoutingHint,
|
||||
CustomPricingLiteLLMParams,
|
||||
ModelResponseStream,
|
||||
ProviderRoutingHints,
|
||||
RawRequestTypedDict,
|
||||
|
|
@ -1115,7 +1114,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o
|
|||
)
|
||||
from litellm.responses.bridge_check import (
|
||||
bridge_reasoning_effort,
|
||||
keep_registered_mode,
|
||||
str_keyed_mapping,
|
||||
)
|
||||
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
|
||||
|
|
@ -1232,93 +1230,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]:
|
|||
return values
|
||||
|
||||
|
||||
def _build_custom_pricing_entry(
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
model_info: dict | None = None,
|
||||
) -> dict:
|
||||
"""Build a complete model cost entry from kwargs and model_info.
|
||||
|
||||
Collects all CustomPricingLiteLLMParams fields present in kwargs and
|
||||
merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
|
||||
so that register_model() receives the full pricing configuration.
|
||||
"""
|
||||
entry: Final[dict] = {"litellm_provider": custom_llm_provider}
|
||||
|
||||
for field_name in CustomPricingLiteLLMParams.model_fields:
|
||||
value = kwargs.get(field_name)
|
||||
if value is not None:
|
||||
entry[field_name] = value
|
||||
|
||||
if model_info and isinstance(model_info, dict):
|
||||
for key in ("mode", "supports_prompt_caching", "max_tokens"):
|
||||
if key in model_info and model_info[key] is not None:
|
||||
entry.setdefault(key, model_info[key])
|
||||
|
||||
return entry
|
||||
|
||||
|
||||
def _get_router_deployment_id(kwargs: dict) -> str | None:
|
||||
for metadata_key in ("litellm_metadata", "metadata"):
|
||||
metadata = kwargs.get(metadata_key) or {}
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
deployment_model_info = metadata.get("model_info") or {}
|
||||
if not isinstance(deployment_model_info, dict):
|
||||
continue
|
||||
deployment_id = deployment_model_info.get("id")
|
||||
if deployment_id is not None:
|
||||
return str(deployment_id)
|
||||
return None
|
||||
|
||||
|
||||
def _register_custom_pricing_for_request(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
model_info: dict | None,
|
||||
) -> None:
|
||||
"""Register per-request custom pricing in litellm.model_cost.
|
||||
|
||||
Router-originated requests (identified by the deployment id the router puts
|
||||
in metadata) get their full pricing registered under that unique id only;
|
||||
the shared ``{provider}/{model}`` key receives the entry with pricing fields
|
||||
stripped, mirroring Router._create_deployment. This keeps one deployment's
|
||||
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
|
||||
pricing used by sibling deployments of the same backend model. Direct SDK
|
||||
calls keep the legacy behavior of registering the shared key with pricing.
|
||||
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
|
||||
deployment registration, since a call routes by its own ``model_info``.
|
||||
"""
|
||||
entry: Final = _build_custom_pricing_entry(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
shared_key: Final = f"{custom_llm_provider}/{model}"
|
||||
deployment_id: Final = _get_router_deployment_id(kwargs)
|
||||
if deployment_id is None:
|
||||
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
|
||||
return
|
||||
shared_entry: Final = keep_registered_mode(
|
||||
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
|
||||
)
|
||||
litellm.register_model(
|
||||
{deployment_id: entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
litellm.register_model(
|
||||
{shared_key: shared_entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
|
||||
|
||||
def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None:
|
||||
return ctx.metadata
|
||||
|
||||
|
|
@ -5750,19 +5661,6 @@ def completion(
|
|||
"aws_region_name", None
|
||||
) # support region-based pricing for bedrock
|
||||
|
||||
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
|
||||
if (
|
||||
(input_cost_per_token is not None and output_cost_per_token is not None)
|
||||
or input_cost_per_second is not None
|
||||
or output_cost_per_second is not None
|
||||
or cost_per_second is not None
|
||||
):
|
||||
_register_custom_pricing_for_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
|
||||
custom_prompt_dict = {}
|
||||
if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token:
|
||||
|
|
@ -6705,12 +6603,6 @@ def embedding(
|
|||
headers.update(proxy_headers)
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
|
||||
### CUSTOM MODEL COST ###
|
||||
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
|
||||
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
|
||||
cost_per_second: Final = kwargs.get("cost_per_second", None)
|
||||
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
|
||||
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
|
||||
openai_params: Final = [
|
||||
"user",
|
||||
"dimensions",
|
||||
|
|
@ -6757,20 +6649,6 @@ def embedding(
|
|||
**non_default_params,
|
||||
)
|
||||
|
||||
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
|
||||
if (
|
||||
(input_cost_per_token is not None and output_cost_per_token is not None)
|
||||
or input_cost_per_second is not None
|
||||
or output_cost_per_second is not None
|
||||
or cost_per_second is not None
|
||||
):
|
||||
_register_custom_pricing_for_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=kwargs.get("model_info"),
|
||||
)
|
||||
|
||||
logging: Final[LiteLLMLoggingObj] = litellm_logging_obj
|
||||
logging.update_environment_variables(
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError
|
|||
from typing_extensions import NotRequired, ReadOnly, Required, TypedDict, Unpack
|
||||
|
||||
import litellm
|
||||
from litellm._catalog import is_priced
|
||||
from litellm._internal_context import with_service_target
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.dual_cache import DualCache, LimitedSizeOrderedDict
|
||||
|
|
@ -605,7 +606,7 @@ def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None)
|
|||
# See: https://github.com/BerriAI/litellm/issues/24770
|
||||
safe_name = str(model_name).replace("\n", "").replace("\r", "")
|
||||
served_deployments = _deployments_served_for(model_name, llm_router)
|
||||
if not _is_cost_explicitly_configured(served_deployments):
|
||||
if not (deployment_prices := tuple(_priced_deployment_infos(served_deployments))):
|
||||
verbose_proxy_logger.debug(
|
||||
"Model %s has zero cost but no explicit cost "
|
||||
"configuration in model_cost entry — treating as unknown "
|
||||
|
|
@ -616,7 +617,11 @@ def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None)
|
|||
zero_cost_cache[model_name] = False
|
||||
return False
|
||||
|
||||
if _has_explicit_positive_cost(served_deployments):
|
||||
if any(
|
||||
_is_positive_cost(info.get("input_cost_per_token"))
|
||||
or _is_positive_cost(info.get("output_cost_per_token"))
|
||||
for info in deployment_prices
|
||||
):
|
||||
verbose_proxy_logger.debug(
|
||||
"Model %s routes to a deployment with an explicit positive per-token price (enforce budget)",
|
||||
safe_name,
|
||||
|
|
@ -696,47 +701,12 @@ def _has_ptu_flat_cost(deployments: Sequence[DeploymentTypedDict]) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _is_cost_explicitly_configured(deployments: Sequence[DeploymentTypedDict]) -> bool:
|
||||
"""
|
||||
Check if any of the deployments has cost fields explicitly set in its
|
||||
litellm.model_cost entry.
|
||||
|
||||
When Router._create_deployment() registers a model not in the global
|
||||
cost map, it creates a sparse entry like {"id": "<hash>"} with no cost
|
||||
fields. _get_model_info_helper() then defaults missing costs to 0.
|
||||
This function detects that scenario by checking the raw model_cost entry.
|
||||
|
||||
It also reaches a deployment that prices itself through its ``model_info`` block, whose entry
|
||||
lands in the cost map under the deployment id.
|
||||
"""
|
||||
def _priced_deployment_infos(deployments: Sequence[DeploymentTypedDict]) -> Iterator[Mapping[str, object]]:
|
||||
for deployment in deployments:
|
||||
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
|
||||
if model_id is None:
|
||||
continue
|
||||
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
|
||||
if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> bool:
|
||||
"""Whether any of the deployments carries an explicit positive per-token price.
|
||||
|
||||
The group's price is read from the deployments named after the routed group and from
|
||||
that group's own alias target, while an alias chain is served from the deployments named
|
||||
after its routed group or from a wildcard route; a priced route among those serves the
|
||||
request at its own rate whatever the group's price reads.
|
||||
"""
|
||||
for deployment in deployments:
|
||||
model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
|
||||
if model_id is None:
|
||||
continue
|
||||
raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
|
||||
if _is_positive_cost(raw_entry.get("input_cost_per_token")) or _is_positive_cost(
|
||||
raw_entry.get("output_cost_per_token")
|
||||
if isinstance(model_id := (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id"), str) and is_priced(
|
||||
model_id
|
||||
):
|
||||
return True
|
||||
return False
|
||||
yield litellm.get_model_info(model=model_id)
|
||||
|
||||
|
||||
_EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
|
@ -798,7 +768,7 @@ def model_has_no_cost_mapping(model: str | None, llm_router: Router | None) -> b
|
|||
if _model_group_has_pricing(model=model, llm_router=llm_router):
|
||||
return False
|
||||
|
||||
return not _is_cost_explicitly_configured(llm_router.get_model_list(model_name=model) or ())
|
||||
return not any(_priced_deployment_infos(llm_router.get_model_list(model_name=model) or ()))
|
||||
|
||||
|
||||
def _unpriced_models_in_request(model: str | list[str] | None, llm_router: Router | None) -> tuple[str, ...]:
|
||||
|
|
|
|||
|
|
@ -203,7 +203,7 @@ from litellm.types.utils import (
|
|||
is_server_derived_pricing_key,
|
||||
pricing_override_fields,
|
||||
)
|
||||
from litellm.utils import cost_map_omits_token_price, load_credentials_from_list, with_named_credential
|
||||
from litellm.utils import load_credentials_from_list, with_named_credential
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from aiohttp import ClientSession
|
||||
|
|
@ -15996,7 +15996,11 @@ def _enrich_model_info_with_litellm_data(
|
|||
discovered_model_info: Final = (
|
||||
llm_router.get_discovered_model_info(model_info.get("id")) if llm_router is not None else MappingProxyType({})
|
||||
)
|
||||
unpriced: Final = cost_map_omits_token_price(model_info.get("id"), litellm_model_info.get("key"))
|
||||
from litellm._catalog import is_priced
|
||||
|
||||
unpriced: Final = not any(
|
||||
is_priced(key) for key in (model_info.get("id"), litellm_model_info.get("key")) if isinstance(key, str)
|
||||
)
|
||||
stamped_model_info: Final = MappingProxyType(
|
||||
{**model_info, **_pricing_override_stamps(model_info, model.get("litellm_params") or MappingProxyType({}))}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ from typing_extensions import assert_never, overload
|
|||
import litellm
|
||||
import litellm.litellm_core_utils.exception_mapping_utils
|
||||
from litellm import get_secret_str
|
||||
from litellm._catalog import is_priced
|
||||
from litellm._internal_context import service_target, with_service_target
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm._uuid import uuid
|
||||
|
|
@ -11548,7 +11549,7 @@ class Router:
|
|||
|
||||
A strategy-router alias is never the deployment actually called or
|
||||
billed, so custom pricing configured on it must not become a cost-map
|
||||
price: an explicit zero would let ``_is_cost_explicitly_configured``
|
||||
price: an explicit zero would let the free-model budget policy
|
||||
treat the alias as a genuinely free model and waive budget checks for
|
||||
requests that route to (and bill as) a real deployment.
|
||||
"""
|
||||
|
|
@ -12321,7 +12322,7 @@ class Router:
|
|||
2. If not, check if litellm model name is in model info
|
||||
3. If not, return None
|
||||
"""
|
||||
from litellm.utils import cost_map_omits_token_price, update_dictionary
|
||||
from litellm.utils import update_dictionary
|
||||
|
||||
model_info: ModelInfo | None = None
|
||||
custom_model_info: dict | None = None
|
||||
|
|
@ -12386,7 +12387,7 @@ class Router:
|
|||
builtin_key: Final = (
|
||||
litellm_model_name_model_info.get("key") if litellm_model_name_model_info is not None else None
|
||||
)
|
||||
if cost_map_omits_token_price(model_id, builtin_key, base_model_key):
|
||||
if not any(is_priced(key) for key in (model_id, builtin_key, base_model_key) if isinstance(key, str)):
|
||||
return cast( # cast-ok: TypedDict spread with overridden keys loses its type
|
||||
ModelInfo, {**model_info, "input_cost_per_token": None, "output_cost_per_token": None}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -3660,22 +3660,6 @@ def set_live_deployment_replay(replay: Callable[[], None]) -> None:
|
|||
_LiveDeploymentReplay.callbacks = (replay,)
|
||||
|
||||
|
||||
def cost_map_omits_token_price(*keys: object) -> bool:
|
||||
"""Whether the raw ``litellm.model_cost`` entries under ``keys`` exist but none carries a per-token price.
|
||||
|
||||
``get_model_info`` substitutes 0 for a missing price, which reads exactly like a declared
|
||||
zero. Surfaces that report pricing use this to keep an unpriced deployment at ``None``.
|
||||
"""
|
||||
entries: Final = tuple(
|
||||
entry
|
||||
for entry in (litellm.model_cost.get(key) for key in keys if isinstance(key, str))
|
||||
if isinstance(entry, dict)
|
||||
)
|
||||
return len(entries) > 0 and not any(
|
||||
"input_cost_per_token" in entry or "output_cost_per_token" in entry for entry in entries
|
||||
)
|
||||
|
||||
|
||||
def register_model(
|
||||
model_cost: str | dict,
|
||||
*,
|
||||
|
|
@ -6322,25 +6306,8 @@ def get_model_info_helper(
|
|||
|
||||
if _model_info is None or key is None:
|
||||
raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider))
|
||||
_input_cost_per_token: float | None = _model_info.get("input_cost_per_token")
|
||||
if _input_cost_per_token is None:
|
||||
# default value to 0, be noisy about this
|
||||
verbose_logger.debug(
|
||||
"model=%s, custom_llm_provider=%s has no input_cost_per_token in model_cost_map. Defaulting to 0.",
|
||||
model,
|
||||
custom_llm_provider,
|
||||
)
|
||||
_input_cost_per_token = 0
|
||||
|
||||
_output_cost_per_token: float | None = _model_info.get("output_cost_per_token")
|
||||
if _output_cost_per_token is None:
|
||||
# default value to 0, be noisy about this
|
||||
verbose_logger.debug(
|
||||
"model=%s, custom_llm_provider=%s has no output_cost_per_token in model_cost_map. Defaulting to 0.",
|
||||
model,
|
||||
custom_llm_provider,
|
||||
)
|
||||
_output_cost_per_token = 0
|
||||
_input_cost_per_token: Final = _model_info.get("input_cost_per_token")
|
||||
_output_cost_per_token: Final = _model_info.get("output_cost_per_token")
|
||||
|
||||
returned_model_info: Final = ModelInfoBase(
|
||||
key=key,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,90 @@ from litellm.rust_bridge._native import ModelNotFound, ResolvedModel
|
|||
from litellm.utils import ModelNotMappedError, _invalidate_model_cost_lowercase_map, get_model_cost_key
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prices", "expected"),
|
||||
(
|
||||
({}, False),
|
||||
({"input_cost_per_token": None, "output_cost_per_token": None}, False),
|
||||
({"input_cost_per_token": 0, "output_cost_per_token": 0}, True),
|
||||
({"input_cost_per_token": 0.25}, True),
|
||||
({"output_cost_per_token": 0.5}, True),
|
||||
({"input_cost_per_token": -1}, False),
|
||||
({"input_cost_per_token": "invalid-price"}, False),
|
||||
({"input_cost_per_second": 1.0}, False),
|
||||
({"cache_read_input_token_cost": 0.0}, True),
|
||||
({"cache_creation_input_token_cost_above_1hr": 0.0}, True),
|
||||
({"output_cost_per_reasoning_token": 0.5}, True),
|
||||
({"input_cost_per_audio_token": 0.5}, True),
|
||||
({"output_cost_per_image_token": 0.5}, True),
|
||||
({"input_cost_per_token_priority": 0.5}, True),
|
||||
({"input_cost_per_token_batches": 0.5}, True),
|
||||
({"input_cost_per_token_above_128k_tokens": 0.5}, True),
|
||||
({"tiered_pricing": [{"range": [0, 200], "input_cost_per_token": 0.5}]}, True),
|
||||
({"tiered_pricing": [{"range": [100, 200], "input_cost_per_token": 0.5}]}, False),
|
||||
({"off_peak_pricing": {"hours_utc": "01:00-02:00", "input_cost_per_token": 0.5}}, True),
|
||||
(
|
||||
{
|
||||
"input_cost_per_token": 1.0,
|
||||
"tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 2.0}],
|
||||
},
|
||||
True,
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_is_priced_distinguishes_explicit_token_rates(
|
||||
monkeypatch: pytest.MonkeyPatch, prices: dict[str, object], expected: bool
|
||||
) -> None:
|
||||
from litellm._catalog import is_priced
|
||||
|
||||
monkeypatch.setattr(litellm, "model_cost", {"openai/price-probe": {"litellm_provider": "openai", **prices}})
|
||||
assert is_priced("price-probe", "openai") is expected
|
||||
assert is_priced("unmapped-price-probe", "openai") is False
|
||||
|
||||
|
||||
def test_is_priced_tracks_selected_live_row_and_tombstones(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm._catalog import is_priced
|
||||
|
||||
rows: Final[dict[str, dict[str, object]]] = {
|
||||
"price-probe": {"litellm_provider": "openai", "input_cost_per_token": 1},
|
||||
"openai/price-probe": {"litellm_provider": "openai"},
|
||||
}
|
||||
monkeypatch.setattr(litellm, "model_cost", rows)
|
||||
assert is_priced("price-probe", "openai") is False
|
||||
rows["openai/price-probe"]["input_cost_per_token"] = 0
|
||||
assert is_priced("price-probe", "openai") is True
|
||||
rows["openai/price-probe"] = {"litellm_provider": "openai"}
|
||||
assert is_priced("price-probe", "openai") is False
|
||||
del rows["openai/price-probe"]
|
||||
assert is_priced("price-probe", "openai") is True
|
||||
del rows["price-probe"]
|
||||
assert is_priced("price-probe", "openai") is False
|
||||
|
||||
|
||||
def test_providerless_deployment_prices_without_writing_provider_context(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm._catalog import is_priced
|
||||
|
||||
row: Final = {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}
|
||||
monkeypatch.setitem(litellm.model_cost, "providerless-priced-deployment", row)
|
||||
assert is_priced("providerless-priced-deployment", "openai") is True
|
||||
assert litellm.cost_per_token(
|
||||
"providerless-priced-deployment", 10, 20, custom_llm_provider="openai"
|
||||
) == pytest.approx((10.0, 40.0))
|
||||
assert row == {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", (None, "", 123))
|
||||
def test_presence_does_not_repair_an_explicit_invalid_provider(
|
||||
monkeypatch: pytest.MonkeyPatch, provider: object
|
||||
) -> None:
|
||||
from litellm._catalog import is_priced
|
||||
|
||||
row: Final = {"litellm_provider": provider, "input_cost_per_token": 1.0, "output_cost_per_token": 2.0}
|
||||
monkeypatch.setitem(litellm.model_cost, "openai/invalid-provider-price-probe", row)
|
||||
assert is_priced("invalid-provider-price-probe", "openai") is False
|
||||
assert row == {"litellm_provider": provider, "input_cost_per_token": 1.0, "output_cost_per_token": 2.0}
|
||||
|
||||
|
||||
def test_model_info_preserves_every_bundled_row() -> None:
|
||||
for key, row in litellm.model_cost.items():
|
||||
result: Final = _read.resolve(key)
|
||||
|
|
|
|||
309
tests/unit/_cost/test_price.py
Normal file
309
tests/unit/_cost/test_price.py
Normal file
|
|
@ -0,0 +1,309 @@
|
|||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm._cost import billing_context, evaluate, price, require_priced, token_usage
|
||||
from litellm._internal_context import pinned_billing_time
|
||||
from litellm.exceptions import ModelNotMappedError
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.rust_bridge import _native
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tier_prices", "expected"),
|
||||
(
|
||||
({"output_cost_per_token": 4.0}, (10.0, 20.0)),
|
||||
({"input_cost_per_token": None, "output_cost_per_token": 4.0}, None),
|
||||
({"input_cost_per_token": 0.0, "output_cost_per_token": 4.0}, (0.0, 40.0)),
|
||||
({"input_cost_per_token": 3.0}, (30.0, 20.0)),
|
||||
({"input_cost_per_token": 3.0, "output_cost_per_token": None}, None),
|
||||
({"input_cost_per_token": 3.0, "output_cost_per_token": 0.0}, (30.0, 0.0)),
|
||||
),
|
||||
ids=("input-absent", "input-null", "input-zero", "output-absent", "output-null", "output-zero"),
|
||||
)
|
||||
def test_tier_declared_null_is_unpriced_and_explicit_zero_remains_free(
|
||||
monkeypatch: pytest.MonkeyPatch, tier_prices: Mapping[str, object], expected: tuple[float, float] | None
|
||||
) -> None:
|
||||
model: Final = "openai/m5-tier-presence"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 2.0,
|
||||
"tiered_pricing": [{"range": [0, 100], **tier_prices}],
|
||||
},
|
||||
)
|
||||
response: Final = ModelResponse(model=model, usage=Usage(prompt_tokens=10, completion_tokens=10))
|
||||
if expected is None:
|
||||
with pytest.raises(ModelNotMappedError):
|
||||
litellm.cost_per_token(model, 10, 10, custom_llm_provider="openai")
|
||||
with pytest.raises(ModelNotMappedError):
|
||||
litellm.completion_cost(response, model=model, custom_llm_provider="openai")
|
||||
else:
|
||||
assert litellm.cost_per_token(model, 10, 10, custom_llm_provider="openai") == pytest.approx(expected)
|
||||
assert litellm.completion_cost(response, model=model, custom_llm_provider="openai") == pytest.approx(
|
||||
sum(expected)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("rates", "usage", "expected"),
|
||||
(
|
||||
(
|
||||
{
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 2.0,
|
||||
"tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 3.0, "output_cost_per_token": 4.0}],
|
||||
},
|
||||
Usage(prompt_tokens=10, completion_tokens=10),
|
||||
(30.0, 40.0),
|
||||
),
|
||||
(
|
||||
{
|
||||
"cache_read_input_token_cost": 9.0,
|
||||
"tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 1.0, "output_cost_per_token": 2.0}],
|
||||
},
|
||||
Usage(prompt_tokens=100, completion_tokens=10, prompt_tokens_details={"cached_tokens": 40}),
|
||||
(100.0, 20.0),
|
||||
),
|
||||
(
|
||||
{
|
||||
"cache_creation_input_token_cost_above_1hr": 9.0,
|
||||
"tiered_pricing": [
|
||||
{
|
||||
"range": [0, 100],
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 2.0,
|
||||
"cache_creation_input_token_cost": 3.0,
|
||||
}
|
||||
],
|
||||
},
|
||||
Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=10,
|
||||
prompt_tokens_details={
|
||||
"cache_creation_tokens": 20,
|
||||
"cache_creation_token_details": {"ephemeral_5m_input_tokens": 10, "ephemeral_1h_input_tokens": 10},
|
||||
},
|
||||
),
|
||||
(140.0, 20.0),
|
||||
),
|
||||
),
|
||||
ids=("flat-and-tier", "tier-cache-read-fallback", "tier-hourly-cache-write-fallback"),
|
||||
)
|
||||
def test_public_token_pricing_preserves_tier_local_fallbacks(
|
||||
monkeypatch: pytest.MonkeyPatch, rates: Mapping[str, object], usage: Usage, expected: tuple[float, float]
|
||||
) -> None:
|
||||
model: Final = "openai/m5-tier-fallback"
|
||||
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **rates})
|
||||
assert litellm.cost_per_token(
|
||||
model, usage.prompt_tokens, usage.completion_tokens, custom_llm_provider="openai", usage_object=usage
|
||||
) == pytest.approx(expected)
|
||||
assert litellm.completion_cost(
|
||||
ModelResponse(model=model, usage=usage), model=model, custom_llm_provider="openai"
|
||||
) == pytest.approx(sum(expected))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
@pytest.mark.parametrize(
|
||||
("kind", "rates", "expected"),
|
||||
(
|
||||
("unknown", None, None),
|
||||
("metadata", {}, None),
|
||||
("free", {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, 0.0),
|
||||
("paid", {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}, 50.0),
|
||||
),
|
||||
)
|
||||
def test_unknown_free_and_priced_public_surfaces(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
stream: bool,
|
||||
kind: str,
|
||||
rates: Mapping[str, float] | None,
|
||||
expected: float | None,
|
||||
) -> None:
|
||||
model: Final = f"openai/m5-{kind}"
|
||||
if rates is not None:
|
||||
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **rates})
|
||||
info: Final = litellm.get_model_info(model, custom_llm_provider="openai")
|
||||
assert info["input_cost_per_token"] == rates.get("input_cost_per_token")
|
||||
assert info["output_cost_per_token"] == rates.get("output_cost_per_token")
|
||||
|
||||
usage: Final = Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30)
|
||||
response: Final = ModelResponse(model=model, usage=usage)
|
||||
logging: Final = Logging(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
stream=stream,
|
||||
call_type="completion",
|
||||
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
litellm_call_id="m5-price-surfaces",
|
||||
function_id="m5-price-surfaces",
|
||||
)
|
||||
logging.update_environment_variables(litellm_params={}, optional_params={}, custom_llm_provider="openai")
|
||||
if expected is None:
|
||||
with pytest.raises(ModelNotMappedError):
|
||||
litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai", usage_object=usage)
|
||||
with pytest.raises(ModelNotMappedError):
|
||||
litellm.completion_cost(response, model=model, custom_llm_provider="openai")
|
||||
assert logging.response_cost_calculator(response) is None
|
||||
assert logging.model_call_details.get("response_cost_failure_debug_information") is not None
|
||||
else:
|
||||
split: Final = litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai", usage_object=usage)
|
||||
assert split == pytest.approx((expected / 5, expected * 4 / 5))
|
||||
assert litellm.completion_cost(response, model=model, custom_llm_provider="openai") == pytest.approx(expected)
|
||||
assert logging.response_cost_calculator(response) == pytest.approx(expected)
|
||||
|
||||
|
||||
def test_public_pricing_reads_direct_row_updates(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
model: Final = "openai/m5-live-update"
|
||||
row: Final = {"litellm_provider": "openai", "input_cost_per_token": 1.0, "output_cost_per_token": 2.0}
|
||||
monkeypatch.setitem(litellm.model_cost, model, row)
|
||||
assert litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai") == pytest.approx((10.0, 40.0))
|
||||
monkeypatch.setitem(row, "input_cost_per_token", 3.0)
|
||||
assert litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai") == pytest.approx((30.0, 40.0))
|
||||
|
||||
|
||||
def test_invalid_price_is_not_authorized_by_permissive_model_info(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
model: Final = "openai/m5-invalid-row"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
"litellm_provider": "openai",
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 2.0,
|
||||
"input_cost_per_audio_token": "invalid",
|
||||
},
|
||||
)
|
||||
assert litellm.get_model_info(model, custom_llm_provider="openai")["input_cost_per_token"] == 1.0
|
||||
with pytest.raises(ModelNotMappedError, match="input_cost_per_audio_token"):
|
||||
litellm.cost_per_token(model, 10, 20, custom_llm_provider="openai")
|
||||
|
||||
|
||||
def test_custom_cost_uses_ephemeral_row_and_native_cache_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
model: Final = "openai/m5-custom-cost"
|
||||
row: Final = {"litellm_provider": "openai", "input_cost_per_token": 10.0, "output_cost_per_token": 20.0}
|
||||
monkeypatch.setitem(litellm.model_cost, model, row)
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=10,
|
||||
prompt_tokens_details={"cached_tokens": 40, "cache_creation_tokens": 20},
|
||||
)
|
||||
assert litellm.cost_per_token(
|
||||
model,
|
||||
100,
|
||||
10,
|
||||
usage_object=usage,
|
||||
custom_cost_per_token={"input_cost_per_token": 1.0, "output_cost_per_token": 2.0},
|
||||
) == pytest.approx((100.0, 20.0))
|
||||
assert litellm.model_cost[model] is row
|
||||
assert row["input_cost_per_token"] == 10.0
|
||||
assert litellm.cost_per_token(model, 100, 10, custom_llm_provider="openai") == pytest.approx((1000.0, 200.0))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tier", ("priority", "fast", " FAST "))
|
||||
def test_billing_context_normalizes_priority_alias(tier: str) -> None:
|
||||
row: Final = {"litellm_provider": "openai", "input_cost_per_token": 1.0, "input_cost_per_token_priority": 3.0}
|
||||
usage: Final = Usage(prompt_tokens=10, completion_tokens=0)
|
||||
result: Final = require_priced(
|
||||
price(row, token_usage(usage, row), billing_context(datetime(2026, 1, 1), tier)),
|
||||
"priority",
|
||||
)
|
||||
assert (result.input, result.output) == (30.0, 0.0)
|
||||
|
||||
|
||||
def test_adapter_preserves_missing_meter_and_native_fallback_result() -> None:
|
||||
when: Final = billing_context(datetime(2026, 1, 1))
|
||||
missing: Final = price({"litellm_provider": "openai"}, _native.UsageInput({"input_cost_per_token": 1}), when)
|
||||
assert isinstance(missing, _native.Unpriced)
|
||||
assert tuple(missing.missing) == ("input_cost_per_token",)
|
||||
fallback: Final = require_priced(
|
||||
price(
|
||||
{"litellm_provider": "openai", "input_cost_per_token": 2.0},
|
||||
_native.UsageInput({"cache_read_input_token_cost": 10}),
|
||||
when,
|
||||
),
|
||||
"cache fallback",
|
||||
)
|
||||
assert fallback.input == 20.0
|
||||
assert fallback.lines[0].fallback_from is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("factor", (1.0, 1.1))
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "completion", "cache", "tier", "expected", "rates"),
|
||||
(
|
||||
(10, 10, 0, None, (10, 20), (1, 2)),
|
||||
(128000, 128000, 0, None, (128000, 256000), (1, 2)),
|
||||
(128001, 10, 0, None, (384003, 20), (3, 2)),
|
||||
(10, 128001, 0, None, (10, 512004), (1, 4)),
|
||||
(128001, 128001, 0, None, (384003, 512004), (3, 4)),
|
||||
(100, 10, 80, None, (100, 20), (1, 2)),
|
||||
(100, 10, 0, "priority", (100, 20), (1, 2)),
|
||||
),
|
||||
)
|
||||
def test_vertex_directional_route_uses_real_aggregate_quantities_and_native_projected_rates(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
factor: float,
|
||||
prompt: int,
|
||||
completion: int,
|
||||
cache: int,
|
||||
tier: str | None,
|
||||
expected: tuple[float, float],
|
||||
rates: tuple[float, float],
|
||||
) -> None:
|
||||
model: Final = "vertex_ai/gemini-2-directional-test"
|
||||
row: Final = {
|
||||
"litellm_provider": "vertex_ai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 2.0,
|
||||
"input_cost_per_token_above_128k_tokens": 3.0,
|
||||
"output_cost_per_token_above_128k_tokens": 4.0,
|
||||
"cache_read_input_token_cost": 0.1,
|
||||
"input_cost_per_token_priority": 5.0,
|
||||
"output_cost_per_token_priority": 6.0,
|
||||
"regional_endpoint_uplift_multiplier": factor,
|
||||
"off_peak_pricing": {"hours_utc": "00:00-06:00", "input_cost_per_token": 0.1, "output_cost_per_token": 0.2},
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, model, row)
|
||||
shaped: Final = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai")
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=prompt, completion_tokens=completion, prompt_tokens_details={"cached_tokens": cache}
|
||||
)
|
||||
when: Final = datetime(2026, 1, 1, 1, tzinfo=timezone.utc)
|
||||
with pinned_billing_time(when):
|
||||
actual: Final = litellm.cost_per_token(
|
||||
model,
|
||||
prompt,
|
||||
completion,
|
||||
custom_llm_provider="vertex_ai",
|
||||
usage_object=usage,
|
||||
service_tier=tier,
|
||||
vertex_location="us-east5",
|
||||
)
|
||||
assert actual == pytest.approx((expected[0] * factor, expected[1] * factor))
|
||||
for source in (row, shaped):
|
||||
evaluated: Final = evaluate(
|
||||
source,
|
||||
_native.UsageInput(
|
||||
{"input_cost_per_token": prompt, "output_cost_per_token": completion}, prompt_tokens=prompt
|
||||
),
|
||||
billing_context(when, tier),
|
||||
project=("input_cost_per_token", "output_cost_per_token"),
|
||||
factors=(factor,),
|
||||
directional_token_threshold=128000,
|
||||
)
|
||||
assert isinstance(evaluated, _native.PriceEvaluation)
|
||||
priced: Final = require_priced(evaluated.pricing, model)
|
||||
assert (priced.input, priced.output) == pytest.approx(actual)
|
||||
assert tuple(selected.unit_price for selected in evaluated.rates) == pytest.approx(
|
||||
(rates[0] * factor, rates[1] * factor)
|
||||
)
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -11,13 +11,14 @@ Tests the cost calculation for Dashscope models including:
|
|||
import math
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
# Add the project root to Python path
|
||||
import litellm
|
||||
from litellm.exceptions import ModelNotMappedError
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
||||
resolve_explicit_cache_read_cost_rate,
|
||||
calculate_prompt_caching_savings,
|
||||
generic_cost_per_token,
|
||||
get_token_type_cost_breakdown,
|
||||
|
|
@ -43,32 +44,35 @@ class TestDashscopeCostCalculator:
|
|||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompt_tokens_details",
|
||||
("prompt_tokens_details", "expected"),
|
||||
[
|
||||
{"cached_tokens": 0, "cache_type": "ephemeral"},
|
||||
{"cached_tokens": 600},
|
||||
{"cached_tokens": 600, "cache_type": "unknown-future-type"},
|
||||
({"cached_tokens": 0, "cache_type": "ephemeral"}, 0.001),
|
||||
({"cached_tokens": 600}, 0.00052),
|
||||
({"cached_tokens": 600, "cache_type": "unknown-future-type"}, 0.00052),
|
||||
({"cached_tokens": 600, "cache_type": "ephemeral"}, 0.00046),
|
||||
],
|
||||
)
|
||||
def test_explicit_cache_rate_requires_an_ephemeral_cache_hit(
|
||||
self,
|
||||
prompt_tokens_details: dict[str, int | str],
|
||||
):
|
||||
assert (
|
||||
resolve_explicit_cache_read_cost_rate(
|
||||
model_info={
|
||||
"key": "qwen-cache-rate-guard-test",
|
||||
"explicit_cache_read_input_token_cost": 1e-07,
|
||||
},
|
||||
usage=Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
),
|
||||
current_time=None,
|
||||
)
|
||||
is None
|
||||
)
|
||||
expected: float,
|
||||
) -> None:
|
||||
assert generic_cost_per_token(
|
||||
model="qwen-cache-rate-guard-test",
|
||||
custom_llm_provider="dashscope",
|
||||
model_info={
|
||||
"key": "qwen-cache-rate-guard-test",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"explicit_cache_read_input_token_cost": 1e-07,
|
||||
},
|
||||
usage=Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
),
|
||||
current_time=None,
|
||||
) == pytest.approx((expected, 0))
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_info", "current_time", "expected_cache_rate"),
|
||||
|
|
@ -100,7 +104,7 @@ class TestDashscopeCostCalculator:
|
|||
],
|
||||
},
|
||||
None,
|
||||
None,
|
||||
2e-07,
|
||||
),
|
||||
(
|
||||
{
|
||||
|
|
@ -109,7 +113,7 @@ class TestDashscopeCostCalculator:
|
|||
"cache_read_input_token_cost": 2e-07,
|
||||
},
|
||||
None,
|
||||
None,
|
||||
2e-07,
|
||||
),
|
||||
(
|
||||
{
|
||||
|
|
@ -179,7 +183,7 @@ class TestDashscopeCostCalculator:
|
|||
},
|
||||
},
|
||||
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
|
||||
8e-08,
|
||||
None,
|
||||
),
|
||||
(
|
||||
{
|
||||
|
|
@ -191,34 +195,49 @@ class TestDashscopeCostCalculator:
|
|||
},
|
||||
},
|
||||
datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc),
|
||||
1e-07,
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_provider_neutral_cache_rate_resolver_handles_qwen_rate_fallbacks(
|
||||
def test_qwen_cache_mode_uses_native_fallbacks_and_rejects_invalid_rows(
|
||||
self,
|
||||
model_info: ModelInfo,
|
||||
current_time: datetime | None,
|
||||
expected_cache_rate: float | None,
|
||||
):
|
||||
usage = Usage(
|
||||
) -> None:
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=0,
|
||||
prompt_tokens_details={"cached_tokens": 600, "cache_type": "ephemeral"},
|
||||
)
|
||||
|
||||
cache_rate = resolve_explicit_cache_read_cost_rate(
|
||||
if expected_cache_rate is None:
|
||||
with pytest.raises(ModelNotMappedError):
|
||||
generic_cost_per_token(
|
||||
model="qwen-cache-rates",
|
||||
custom_llm_provider="dashscope",
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=current_time,
|
||||
)
|
||||
return
|
||||
cost: Final = generic_cost_per_token(
|
||||
model="qwen-cache-rates",
|
||||
custom_llm_provider="dashscope",
|
||||
model_info=model_info,
|
||||
usage=usage,
|
||||
current_time=current_time,
|
||||
)
|
||||
|
||||
assert cache_rate == expected_cache_rate
|
||||
assert cost == pytest.approx((400e-06 + 600 * expected_cache_rate, 0))
|
||||
|
||||
def test_provider_neutral_cache_rate_resolver_handles_service_tier_rate(self):
|
||||
cache_rate = resolve_explicit_cache_read_cost_rate(
|
||||
def test_ephemeral_cache_uses_the_service_tier_rate(self) -> None:
|
||||
cost: Final = generic_cost_per_token(
|
||||
model="qwen-service-tier-mode-rate-test",
|
||||
custom_llm_provider="dashscope",
|
||||
model_info={
|
||||
"key": "qwen-service-tier-mode-rate-test",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"explicit_cache_read_input_token_cost_priority": 3e-07,
|
||||
},
|
||||
usage=Usage(
|
||||
|
|
@ -230,7 +249,7 @@ class TestDashscopeCostCalculator:
|
|||
service_tier="priority",
|
||||
)
|
||||
|
||||
assert cache_rate == 3e-07
|
||||
assert cost == pytest.approx((0.00058, 0))
|
||||
|
||||
def test_dashscope_flat_pricing_fallback(self):
|
||||
"""
|
||||
|
|
@ -239,9 +258,7 @@ class TestDashscopeCostCalculator:
|
|||
"""
|
||||
usage = Usage(prompt_tokens=1000, completion_tokens=500)
|
||||
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-max", usage=usage
|
||||
)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-max", usage=usage)
|
||||
|
||||
model_info = litellm.get_model_info("dashscope/qwen-max")
|
||||
expected_prompt_cost = 1000 * model_info["input_cost_per_token"]
|
||||
|
|
@ -527,9 +544,7 @@ class TestDashscopeCostCalculator:
|
|||
"""
|
||||
# Tier 1 for qwen-flash is [0, 256,000] tokens
|
||||
usage = Usage(prompt_tokens=100000, completion_tokens=50000)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-flash", usage=usage
|
||||
)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage)
|
||||
|
||||
model_info = litellm.get_model_info("dashscope/qwen-flash")
|
||||
tier_1_pricing = model_info["tiered_pricing"][0]
|
||||
|
|
@ -547,9 +562,7 @@ class TestDashscopeCostCalculator:
|
|||
"""
|
||||
# Tiering for qwen-flash: Tier 1: [0, 256k], Tier 2: [256k, 1M]
|
||||
usage = Usage(prompt_tokens=300000, completion_tokens=300000)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-flash", usage=usage
|
||||
)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage)
|
||||
|
||||
model_info = litellm.get_model_info("dashscope/qwen-flash")
|
||||
tier_1 = model_info["tiered_pricing"][0]
|
||||
|
|
@ -561,9 +574,7 @@ class TestDashscopeCostCalculator:
|
|||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, expected_completion_cost, rel_tol=1e-10)
|
||||
|
||||
graduated_prompt_cost = (256000 * tier_1["input_cost_per_token"]) + (
|
||||
44000 * tier_2["input_cost_per_token"]
|
||||
)
|
||||
graduated_prompt_cost = (256000 * tier_1["input_cost_per_token"]) + (44000 * tier_2["input_cost_per_token"])
|
||||
assert prompt_cost > graduated_prompt_cost
|
||||
|
||||
def test_dashscope_tiered_pricing_boundary_stays_in_lower_tier(self):
|
||||
|
|
@ -572,18 +583,12 @@ class TestDashscopeCostCalculator:
|
|||
official `0 < Token <= 256K` phrasing.
|
||||
"""
|
||||
usage = Usage(prompt_tokens=256000, completion_tokens=1000)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-flash", usage=usage
|
||||
)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-flash", usage=usage)
|
||||
|
||||
tier_1 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][0]
|
||||
|
||||
assert math.isclose(
|
||||
prompt_cost, 256000 * tier_1["input_cost_per_token"], rel_tol=1e-10
|
||||
)
|
||||
assert math.isclose(
|
||||
completion_cost, 1000 * tier_1["output_cost_per_token"], rel_tol=1e-10
|
||||
)
|
||||
assert math.isclose(prompt_cost, 256000 * tier_1["input_cost_per_token"], rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, 1000 * tier_1["output_cost_per_token"], rel_tol=1e-10)
|
||||
|
||||
def test_dashscope_tiered_pricing_output_uses_input_selected_tier(self):
|
||||
"""
|
||||
|
|
@ -595,9 +600,7 @@ class TestDashscopeCostCalculator:
|
|||
|
||||
tier_1 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][0]
|
||||
|
||||
assert math.isclose(
|
||||
completion_cost, 400000 * tier_1["output_cost_per_token"], rel_tol=1e-10
|
||||
)
|
||||
assert math.isclose(completion_cost, 400000 * tier_1["output_cost_per_token"], rel_tol=1e-10)
|
||||
|
||||
def test_dashscope_tiered_pricing_with_caching(self):
|
||||
"""
|
||||
|
|
@ -626,17 +629,13 @@ class TestDashscopeCostCalculator:
|
|||
"""
|
||||
Requests above the highest declared range bill entirely at the last tier's rate.
|
||||
"""
|
||||
usage = Usage(
|
||||
prompt_tokens=1200000, completion_tokens=1000
|
||||
) # Max defined range for qwen-flash is 1M
|
||||
usage = Usage(prompt_tokens=1200000, completion_tokens=1000) # Max defined range for qwen-flash is 1M
|
||||
|
||||
prompt_cost, _ = dashscope_cost_per_token(model="qwen-flash", usage=usage)
|
||||
|
||||
tier_2 = litellm.get_model_info("dashscope/qwen-flash")["tiered_pricing"][1]
|
||||
|
||||
assert math.isclose(
|
||||
prompt_cost, 1200000 * tier_2["input_cost_per_token"], rel_tol=1e-10
|
||||
)
|
||||
assert math.isclose(prompt_cost, 1200000 * tier_2["input_cost_per_token"], rel_tol=1e-10)
|
||||
|
||||
def _register_tiered_model(self, model_key: str, tiered_pricing: list[dict]) -> None:
|
||||
litellm.model_cost[model_key] = {
|
||||
|
|
@ -671,9 +670,7 @@ class TestDashscopeCostCalculator:
|
|||
self._register_string_valued_tiered_model("dashscope/qwen-str-tier-test")
|
||||
|
||||
usage = Usage(prompt_tokens=500, completion_tokens=200)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-str-tier-test", usage=usage
|
||||
)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-str-tier-test", usage=usage)
|
||||
|
||||
assert math.isclose(prompt_cost, 500 * float("4e-07"), rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, 200 * float("1.6e-06"), rel_tol=1e-10)
|
||||
|
|
@ -686,9 +683,7 @@ class TestDashscopeCostCalculator:
|
|||
self._register_string_valued_tiered_model("dashscope/qwen-str-tier-test")
|
||||
|
||||
usage = Usage(prompt_tokens=2500, completion_tokens=3000)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-str-tier-test", usage=usage
|
||||
)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-str-tier-test", usage=usage)
|
||||
|
||||
assert math.isclose(prompt_cost, 2500 * float("8e-07"), rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, 3000 * float("3.2e-06"), rel_tol=1e-10)
|
||||
|
|
@ -721,18 +716,12 @@ class TestDashscopeCostCalculator:
|
|||
usage = Usage(
|
||||
prompt_tokens=300000, # 200k new + 60k cache creation + 40k cache read
|
||||
completion_tokens=1000,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cached_tokens=40000, cache_creation_tokens=60000
|
||||
),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=40000, cache_creation_tokens=60000),
|
||||
)
|
||||
|
||||
prompt_cost, _ = dashscope_cost_per_token(
|
||||
model="qwen-cache-write-test", usage=usage
|
||||
)
|
||||
prompt_cost, _ = dashscope_cost_per_token(model="qwen-cache-write-test", usage=usage)
|
||||
|
||||
expected_prompt_cost = (
|
||||
(200000 * 6.5e-07) + (60000 * 8.125e-07) + (40000 * 6.5e-08)
|
||||
)
|
||||
expected_prompt_cost = (200000 * 6.5e-07) + (60000 * 8.125e-07) + (40000 * 6.5e-08)
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
|
||||
|
|
@ -769,13 +758,9 @@ class TestDashscopeCostCalculator:
|
|||
completion_tokens_details={"reasoning_tokens": 170},
|
||||
)
|
||||
|
||||
prompt_cost, _ = dashscope_cost_per_token(
|
||||
model="qwen-nested-cache-write-test", usage=usage
|
||||
)
|
||||
prompt_cost, _ = dashscope_cost_per_token(model="qwen-nested-cache-write-test", usage=usage)
|
||||
|
||||
assert math.isclose(
|
||||
prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10
|
||||
)
|
||||
assert math.isclose(prompt_cost, (2048 * 5e-07) + (11 * 4e-07), rel_tol=1e-10)
|
||||
|
||||
def test_dashscope_tiered_cache_creation_falls_back_to_tier_input_rate(self):
|
||||
"""
|
||||
|
|
@ -799,9 +784,7 @@ class TestDashscopeCostCalculator:
|
|||
prompt_tokens_details=PromptTokensDetailsWrapper(cache_creation_tokens=4000),
|
||||
)
|
||||
|
||||
prompt_cost, _ = dashscope_cost_per_token(
|
||||
model="qwen-no-cache-write-test", usage=usage
|
||||
)
|
||||
prompt_cost, _ = dashscope_cost_per_token(model="qwen-no-cache-write-test", usage=usage)
|
||||
|
||||
assert math.isclose(prompt_cost, 10000 * 3.25e-07, rel_tol=1e-10)
|
||||
|
||||
|
|
@ -819,18 +802,12 @@ class TestDashscopeCostCalculator:
|
|||
usage = Usage(
|
||||
prompt_tokens=10000,
|
||||
completion_tokens=100,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cached_tokens=2000, cache_creation_tokens=3000
|
||||
),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=2000, cache_creation_tokens=3000),
|
||||
)
|
||||
|
||||
prompt_cost, _ = dashscope_cost_per_token(
|
||||
model="qwen-flat-cache-write-test", usage=usage
|
||||
)
|
||||
prompt_cost, _ = dashscope_cost_per_token(model="qwen-flat-cache-write-test", usage=usage)
|
||||
|
||||
expected_prompt_cost = (
|
||||
(5000 * 3.25e-07) + (3000 * 4.063e-07) + (2000 * 3.25e-08)
|
||||
)
|
||||
expected_prompt_cost = (5000 * 3.25e-07) + (3000 * 4.063e-07) + (2000 * 3.25e-08)
|
||||
|
||||
assert math.isclose(prompt_cost, expected_prompt_cost, rel_tol=1e-10)
|
||||
|
||||
|
|
@ -847,9 +824,7 @@ class TestDashscopeCostCalculator:
|
|||
}
|
||||
|
||||
usage = Usage(prompt_tokens=500, completion_tokens=200)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-input-only-tier-test", usage=usage
|
||||
)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-input-only-tier-test", usage=usage)
|
||||
|
||||
assert math.isclose(prompt_cost, 500 * 4e-07, rel_tol=1e-10)
|
||||
assert math.isclose(completion_cost, 200 * 1.6e-06, rel_tol=1e-10)
|
||||
|
|
@ -872,13 +847,9 @@ class TestDashscopeCostCalculator:
|
|||
completion_tokens=200,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150),
|
||||
)
|
||||
_, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-input-only-reasoning-test", usage=usage
|
||||
)
|
||||
_, completion_cost = dashscope_cost_per_token(model="qwen-input-only-reasoning-test", usage=usage)
|
||||
|
||||
assert math.isclose(
|
||||
completion_cost, (50 * 1.6e-06) + (150 * 4e-06), rel_tol=1e-10
|
||||
)
|
||||
assert math.isclose(completion_cost, (50 * 1.6e-06) + (150 * 4e-06), rel_tol=1e-10)
|
||||
|
||||
def test_dashscope_tier_output_rate_wins_over_the_model_reasoning_rate(self):
|
||||
"""
|
||||
|
|
@ -903,9 +874,7 @@ class TestDashscopeCostCalculator:
|
|||
completion_tokens=200,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150),
|
||||
)
|
||||
_, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-tier-output-reasoning-test", usage=usage
|
||||
)
|
||||
_, completion_cost = dashscope_cost_per_token(model="qwen-tier-output-reasoning-test", usage=usage)
|
||||
|
||||
assert math.isclose(completion_cost, 200 * 1.6e-06, rel_tol=1e-10)
|
||||
|
||||
|
|
@ -927,9 +896,7 @@ class TestDashscopeCostCalculator:
|
|||
completion_tokens=200,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150),
|
||||
)
|
||||
_, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-zero-reasoning-test", usage=usage
|
||||
)
|
||||
_, completion_cost = dashscope_cost_per_token(model="qwen-zero-reasoning-test", usage=usage)
|
||||
|
||||
assert math.isclose(completion_cost, 50 * 1.6e-06, rel_tol=1e-10)
|
||||
|
||||
|
|
@ -956,9 +923,7 @@ class TestDashscopeCostCalculator:
|
|||
completion_tokens=200,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=150),
|
||||
)
|
||||
_, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-tier-zero-reasoning-test", usage=usage
|
||||
)
|
||||
_, completion_cost = dashscope_cost_per_token(model="qwen-tier-zero-reasoning-test", usage=usage)
|
||||
|
||||
assert math.isclose(completion_cost, 50 * 1.6e-06, rel_tol=1e-10)
|
||||
|
||||
|
|
@ -987,9 +952,7 @@ class TestDashscopeCostCalculator:
|
|||
}
|
||||
|
||||
usage = Usage(prompt_tokens=0, completion_tokens=500)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(
|
||||
model="qwen-zero-input-test", usage=usage
|
||||
)
|
||||
prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-zero-input-test", usage=usage)
|
||||
|
||||
assert prompt_cost == 0.0
|
||||
assert math.isclose(completion_cost, 500 * 1.6e-06, rel_tol=1e-10)
|
||||
|
|
|
|||
|
|
@ -1,65 +1,76 @@
|
|||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.constants import FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO
|
||||
from litellm.llms.fireworks_ai.cache_pricing import with_default_cache_read_rate
|
||||
from litellm.types.utils import ModelInfo
|
||||
import litellm
|
||||
from litellm.exceptions import ModelNotMappedError
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token, get_billed_token_rates
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
def test_explicit_cache_read_rate_and_missing_input_rate_keep_the_entry_untouched() -> None:
|
||||
explicit_info: ModelInfo = {"input_cost_per_token": 2e-6, "cache_read_input_token_cost": 1e-6}
|
||||
no_input_rate_info: ModelInfo = {"output_cost_per_token": 3e-6}
|
||||
|
||||
assert with_default_cache_read_rate(explicit_info) is explicit_info
|
||||
assert with_default_cache_read_rate(no_input_rate_info) is no_input_rate_info
|
||||
@pytest.mark.parametrize(
|
||||
("extra", "expected"),
|
||||
(
|
||||
({}, 0.0001),
|
||||
({"cache_read_input_token_cost": None}, 0.0001),
|
||||
({"cache_read_input_token_cost": 0.0}, 0.0),
|
||||
({"cache_read_input_token_cost": 0.3e-6}, 0.00003),
|
||||
({"off_peak_pricing": {"hours_utc": "14:00-00:00", "input_cost_per_token": 1e-6}}, 0.00005),
|
||||
({"off_peak_pricing": {"hours_utc": "14:00-00:00", "output_cost_per_token": 3e-6}}, 0.0001),
|
||||
({"off_peak_pricing": {"hours_utc": "14:00-00:00", "input_cost_per_token": "1e-6"}}, 0.00005),
|
||||
(
|
||||
{
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "14:00-00:00",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"cache_read_input_token_cost": None,
|
||||
}
|
||||
},
|
||||
0.0001,
|
||||
),
|
||||
({"tiered_pricing": [{"range": [0, 200], "input_cost_per_token": 4e-6}]}, 0.0004),
|
||||
),
|
||||
)
|
||||
def test_fireworks_default_cache_rate_matches_projected_rates_without_mutating_the_row(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
extra: Mapping[str, object],
|
||||
expected: float,
|
||||
) -> None:
|
||||
model: Final = "fireworks_ai/cache-default-test"
|
||||
row: Final = {"input_cost_per_token": "2e-6", "litellm_provider": "fireworks_ai", "mode": "chat", **extra}
|
||||
original: Final = deepcopy(row)
|
||||
monkeypatch.setitem(litellm.model_cost, model, row)
|
||||
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="fireworks_ai")
|
||||
usage: Final = Usage(prompt_tokens=100, completion_tokens=0, prompt_tokens_details={"cached_tokens": 100})
|
||||
when: Final = datetime(2026, 1, 1, 16, tzinfo=timezone.utc)
|
||||
assert generic_cost_per_token(
|
||||
model,
|
||||
usage,
|
||||
"fireworks_ai",
|
||||
model_info=model_info,
|
||||
current_time=when,
|
||||
) == pytest.approx((expected, 0))
|
||||
assert get_billed_token_rates(
|
||||
model,
|
||||
"fireworks_ai",
|
||||
usage,
|
||||
model_info=model_info,
|
||||
current_time=when,
|
||||
).cache_read_input_token_cost == pytest.approx(expected / 100)
|
||||
assert row == original
|
||||
|
||||
|
||||
def test_missing_cache_read_rate_is_derived_for_standard_and_off_peak_without_mutating_the_entry() -> None:
|
||||
model_info: ModelInfo = {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "14:00-00:00",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 3e-6,
|
||||
},
|
||||
}
|
||||
original: ModelInfo = deepcopy(model_info)
|
||||
|
||||
derived = with_default_cache_read_rate(model_info)
|
||||
|
||||
assert model_info == original
|
||||
assert derived is not model_info
|
||||
assert derived["cache_read_input_token_cost"] == pytest.approx(2e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO)
|
||||
assert derived["off_peak_pricing"] == {
|
||||
"hours_utc": "14:00-00:00",
|
||||
"input_cost_per_token": 1e-6,
|
||||
"output_cost_per_token": 3e-6,
|
||||
"cache_read_input_token_cost": 1e-6 * FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO,
|
||||
}
|
||||
|
||||
|
||||
def test_off_peak_window_without_its_own_input_rate_reuses_the_standard_derived_rate() -> None:
|
||||
model_info: ModelInfo = {
|
||||
"input_cost_per_token": 2e-6,
|
||||
"off_peak_pricing": {"hours_utc": "14:00-00:00", "output_cost_per_token": 3e-6},
|
||||
}
|
||||
|
||||
derived = with_default_cache_read_rate(model_info)
|
||||
|
||||
assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == derived["cache_read_input_token_cost"]
|
||||
|
||||
|
||||
def test_string_rates_from_config_are_coerced_before_the_discount_is_applied() -> None:
|
||||
model_info: ModelInfo = {
|
||||
"input_cost_per_token": "2e-6",
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "14:00-00:00",
|
||||
"input_cost_per_token": "1e-6",
|
||||
},
|
||||
}
|
||||
|
||||
derived = with_default_cache_read_rate(model_info)
|
||||
|
||||
assert derived["cache_read_input_token_cost"] == pytest.approx(1e-6)
|
||||
assert derived["off_peak_pricing"]["cache_read_input_token_cost"] == pytest.approx(5e-7)
|
||||
def test_fireworks_missing_input_rate_remains_unknown_without_a_cache_price(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
model: Final = "fireworks_ai/cache-unknown-test"
|
||||
row: Final = {"litellm_provider": "fireworks_ai", "mode": "chat", "output_cost_per_token": 3e-6}
|
||||
monkeypatch.setitem(litellm.model_cost, model, row)
|
||||
with pytest.raises(ModelNotMappedError):
|
||||
generic_cost_per_token(
|
||||
model,
|
||||
Usage(prompt_tokens=100, completion_tokens=0, prompt_tokens_details={"cached_tokens": 100}),
|
||||
"fireworks_ai",
|
||||
)
|
||||
assert row == {"litellm_provider": "fireworks_ai", "mode": "chat", "output_cost_per_token": 3e-6}
|
||||
|
|
|
|||
|
|
@ -323,10 +323,8 @@ def test_an_entry_without_an_input_rate_gets_no_cache_read_fallback():
|
|||
}
|
||||
usage: Final = _usage(prompt_tokens=1000, cached_tokens=300, completion_tokens=200)
|
||||
|
||||
prompt_cost, completion_cost = cost_per_token(model="accounts/fireworks/models/no-input-rate-test", usage=usage)
|
||||
|
||||
assert prompt_cost == 0
|
||||
assert completion_cost == 200 * 2e-06
|
||||
with pytest.raises(litellm.ModelNotMappedError, match="input_cost_per_token"):
|
||||
cost_per_token(model="accounts/fireworks/models/no-input-rate-test", usage=usage)
|
||||
|
||||
|
||||
ROUTED_MODEL: Final = next(
|
||||
|
|
|
|||
|
|
@ -7,8 +7,12 @@ per the MCP spec.
|
|||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._experimental.mcp_server.sampling_handler import (
|
||||
_has_priorities,
|
||||
_resolve_model_from_preferences,
|
||||
|
|
@ -30,6 +34,14 @@ def _prefs(*, hints=None, cost=None, speed=None, intelligence=None):
|
|||
)
|
||||
|
||||
|
||||
def test_priority_reader_tolerates_missing_token_prices(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
model: Final = "openai/m5-sampling-reader"
|
||||
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat"})
|
||||
info: Final = litellm.get_model_info(model)
|
||||
assert (info["input_cost_per_token"], info["output_cost_per_token"]) == (None, None)
|
||||
assert _select_model_by_priority([model], _prefs(cost=1.0)) == model
|
||||
|
||||
|
||||
# Model info stubs keyed by model name
|
||||
_MODEL_INFO = {
|
||||
"gpt-3.5-turbo": {
|
||||
|
|
|
|||
|
|
@ -8,17 +8,77 @@ See: https://github.com/BerriAI/litellm/issues/24770
|
|||
"""
|
||||
|
||||
import copy
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.auth.auth_checks import is_model_cost_zero
|
||||
from litellm._catalog import is_priced
|
||||
from litellm.proxy.auth.auth_checks import is_model_cost_zero, model_has_no_cost_mapping
|
||||
from litellm.router import Router
|
||||
|
||||
|
||||
UNPRICED_ZERO_COST_MODEL = "ollama/unpriced-zero-cost-target"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prices", "present", "free"),
|
||||
(
|
||||
({}, False, False),
|
||||
({"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, True, True),
|
||||
({"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}, True, False),
|
||||
({"input_cost_per_token": 0.0}, True, False),
|
||||
({"output_cost_per_token": 2.0}, True, False),
|
||||
({"cache_read_input_token_cost": 0.0}, True, False),
|
||||
(
|
||||
{
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
"ptu_count": 10,
|
||||
"cost_per_ptu_per_hour": 1.0,
|
||||
},
|
||||
True,
|
||||
False,
|
||||
),
|
||||
(
|
||||
{
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 2.0,
|
||||
"tiered_pricing": [{"range": [0, 100], "input_cost_per_token": 3.0, "output_cost_per_token": 4.0}],
|
||||
},
|
||||
True,
|
||||
False,
|
||||
),
|
||||
),
|
||||
ids=(
|
||||
"metadata-only",
|
||||
"explicit-free",
|
||||
"paid",
|
||||
"input-only-zero",
|
||||
"output-only-positive",
|
||||
"cache-only-zero",
|
||||
"ptu",
|
||||
"flat-plus-tier",
|
||||
),
|
||||
)
|
||||
def test_native_price_presence_does_not_replace_free_budget_policy(
|
||||
monkeypatch: pytest.MonkeyPatch, prices: Mapping[str, object], present: bool, free: bool
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost))
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "m5-budget-group",
|
||||
"litellm_params": {"model": "openai/m5-budget-backend", "api_key": "fake-key"},
|
||||
"model_info": {"id": "m5-budget-deployment", **prices},
|
||||
}
|
||||
]
|
||||
)
|
||||
assert is_priced("m5-budget-deployment") is present
|
||||
assert model_has_no_cost_mapping("m5-budget-group", router) is not present
|
||||
assert is_model_cost_zero("m5-budget-group", router) is free
|
||||
|
||||
|
||||
def _explicitly_free(name: str, model: str = "gpt-3.5-turbo", **model_info: float) -> dict:
|
||||
return {
|
||||
"model_name": name,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -25,6 +26,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.budget_reservation import (
|
||||
_get_team_member_budget_counter,
|
||||
estimate_request_input_cost,
|
||||
estimate_request_max_cost,
|
||||
get_budget_window_start,
|
||||
release_unbound_budget_reservation,
|
||||
|
|
@ -197,6 +199,68 @@ def _tiered_deployment(input_cost_per_token: float) -> Deployment:
|
|||
TIERED_BODY: Final = {"model": "tiered-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prices", "expected_input", "expected_max"),
|
||||
(
|
||||
({}, None, None),
|
||||
({"input_cost_per_token": None, "output_cost_per_token": None}, None, None),
|
||||
({"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}, 0.0, 0.0),
|
||||
({"input_cost_per_token": 2.0}, 20.0, None),
|
||||
),
|
||||
ids=("missing", "null", "explicit-zero", "input-only"),
|
||||
)
|
||||
def test_reservation_reader_preserves_unknown_token_prices(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
prices: Mapping[str, float | None],
|
||||
expected_input: float | None,
|
||||
expected_max: float | None,
|
||||
) -> None:
|
||||
model: Final = "openai/m5-reservation-reader"
|
||||
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "openai", "mode": "chat", **prices})
|
||||
request: Final = {"model": model, "messages": [{"role": "user", "content": "hello"}], "max_tokens": 5}
|
||||
assert estimate_request_input_cost(request, "/chat/completions", None, {model: 10}) == expected_input
|
||||
assert estimate_request_max_cost(request, "/chat/completions", None, {model: 10}) == expected_max
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tiered", (False, True))
|
||||
def test_reservation_keeps_the_reasoning_max_and_output_multiplier_without_discounts(
|
||||
monkeypatch: pytest.MonkeyPatch, tiered: bool
|
||||
) -> None:
|
||||
model: Final = "openai/m5-conservative-reservation"
|
||||
prices: Final = {
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 2.0,
|
||||
"output_cost_per_reasoning_token": 5.0,
|
||||
}
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
**({"tiered_pricing": [{"range": [0, 100], **prices}]} if tiered else prices),
|
||||
"input_cost_per_token_flex": 0.1,
|
||||
"output_cost_per_token_flex": 0.2,
|
||||
"off_peak_pricing": {
|
||||
"hours_utc": "00:00-00:00",
|
||||
"input_cost_per_token": 0.01,
|
||||
"output_cost_per_token": 0.02,
|
||||
"output_cost_per_reasoning_token": 0.05,
|
||||
},
|
||||
},
|
||||
)
|
||||
request: Final = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_tokens": 5,
|
||||
"service_tier": "flex",
|
||||
"n": 2,
|
||||
"best_of": 3,
|
||||
}
|
||||
assert estimate_request_input_cost(request, "/chat/completions", None, {model: 10}) == 10.0
|
||||
assert estimate_request_max_cost(request, "/chat/completions", None, {model: 10}) == 85.0
|
||||
|
||||
|
||||
def test_repeated_estimates_reuse_cached_model_cost_info() -> None:
|
||||
router: Final = Router(model_list=[_tiered_deployment(1e-06).model_dump()])
|
||||
first: Final = estimate_request_max_cost(request_body=TIERED_BODY, route="/chat/completions", llm_router=router)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
|
|
@ -137,6 +139,81 @@ def test_compression_savings_priced_at_input_rate():
|
|||
assert result.prompt_caching == 0.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prices", "saved_tokens", "expected"),
|
||||
(
|
||||
({}, 10, 0.0),
|
||||
({"input_cost_per_token": None}, 10, 0.0),
|
||||
({"input_cost_per_token": 0.0}, 10, 0.0),
|
||||
({"input_cost_per_token": 2.0}, -10, 0.0),
|
||||
({"input_cost_per_token": 2.0}, 10, 20.0),
|
||||
),
|
||||
)
|
||||
def test_compression_keeps_the_flat_reporting_estimate_instead_of_token_selection(
|
||||
monkeypatch: pytest.MonkeyPatch, prices: Mapping[str, float | None], saved_tokens: int, expected: float
|
||||
) -> None:
|
||||
model: Final = "openai/compression-policy-test"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
**prices,
|
||||
"output_cost_per_token": 1.0,
|
||||
"tiered_pricing": [{"range": [0, 2000], "input_cost_per_token": 0.1, "output_cost_per_token": 0.1}],
|
||||
"input_cost_per_token_priority": 0.01,
|
||||
"off_peak_pricing": {"hours_utc": "00:00-00:00", "input_cost_per_token": 0.001},
|
||||
},
|
||||
)
|
||||
result: Final = compute_savings_spend(
|
||||
model,
|
||||
"openai",
|
||||
saved_tokens,
|
||||
False,
|
||||
usage_object={"prompt_tokens": 1000, "completion_tokens": 0},
|
||||
cost_breakdown={"service_tier": "priority"},
|
||||
billed_at=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
assert result.compression == expected
|
||||
assert result.prompt_caching == 0.0
|
||||
|
||||
|
||||
def test_compression_reporting_does_not_leak_deployment_prices(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
model: Final = "openai/compression-policy-test"
|
||||
row: Final = {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 1.0,
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, model, dict(row))
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "compression-a",
|
||||
"litellm_params": {"model": model, "input_cost_per_token": 2.0},
|
||||
"model_info": {"id": "compression-a"},
|
||||
},
|
||||
{
|
||||
"model_name": "compression-b",
|
||||
"litellm_params": {"model": model, "input_cost_per_token": 3.0},
|
||||
"model_info": {"id": "compression-b"},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
def estimate(model_id: str) -> float:
|
||||
return compute_savings_spend(
|
||||
model, "openai", 10, False, model_id=model_id, llm_router=lambda: router
|
||||
).compression
|
||||
|
||||
assert estimate("compression-a") == 20.0
|
||||
assert estimate("compression-b") == 30.0
|
||||
assert estimate("compression-a") == 20.0
|
||||
assert litellm.model_cost[model] == row
|
||||
|
||||
|
||||
def test_prompt_caching_savings_priced_at_input_minus_cache_read():
|
||||
input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5")
|
||||
# A model that supports prompt caching must charge less to read from cache;
|
||||
|
|
@ -635,8 +712,6 @@ def test_equal_modeled_usage_is_zero_under_equivalent_model_names() -> None:
|
|||
assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage, usage) == 0.0
|
||||
|
||||
|
||||
|
||||
|
||||
def test_unresolvable_baseline_remains_unknown():
|
||||
usage = _usage(fresh=2000, cached=0, written=0, out=500)
|
||||
assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) is None
|
||||
|
|
|
|||
|
|
@ -13,8 +13,11 @@ from openai.types.completion_usage import CompletionUsage
|
|||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import ModelNotMappedError
|
||||
from litellm import TranscriptionResponse, model_cost
|
||||
from litellm.constants import REALTIME_METERED_USAGE_KEY
|
||||
|
||||
|
||||
from litellm.cost_calculator import BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, batch_cost_calculator, completion_cost, cost_per_token, handle_realtime_stream_cost_calculation, response_cost_calculator
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -43,6 +46,30 @@ import threading
|
|||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", (None, "", 123, {}))
|
||||
def test_registered_bedrock_pricing_does_not_replace_a_declared_provider(
|
||||
monkeypatch: pytest.MonkeyPatch, provider: object
|
||||
) -> None:
|
||||
model: Final = "bedrock-invalid-provider-context"
|
||||
litellm.register_model(
|
||||
{model: {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}},
|
||||
custom_llm_provider="bedrock",
|
||||
persist_across_reloads=False,
|
||||
)
|
||||
monkeypatch.setitem(litellm.model_cost[model], "litellm_provider", provider)
|
||||
if provider == "":
|
||||
assert litellm.cost_per_token(model, 10, 20, custom_llm_provider="bedrock") == (10.0, 40.0)
|
||||
assert litellm.get_model_info(model=model)["litellm_provider"] == ""
|
||||
else:
|
||||
with pytest.raises(ModelNotMappedError, match="litellm_provider"):
|
||||
litellm.cost_per_token(model, 10, 20, custom_llm_provider="bedrock")
|
||||
assert litellm.model_cost[model] == {
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 2.0,
|
||||
"litellm_provider": provider,
|
||||
}
|
||||
|
||||
|
||||
class ResponseCostCapture(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
self.success_cost: float | None = None
|
||||
|
|
@ -493,7 +520,7 @@ def test_completion_cost_skips_huggingface_config_request_with_custom_pricing(
|
|||
assert cost == pytest.approx(usage.prompt_tokens * 0.001 + usage.completion_tokens * 0.002)
|
||||
|
||||
|
||||
def test_completion_cost_returns_zero_for_capability_only_generalization(
|
||||
def test_completion_cost_rejects_unknown_pricing_for_capability_only_generalization(
|
||||
_local_model_cost_map: None,
|
||||
_claude_family_generalization: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
@ -506,13 +533,8 @@ def test_completion_cost_returns_zero_for_capability_only_generalization(
|
|||
usage=Usage(prompt_tokens=17, completion_tokens=9, total_tokens=26),
|
||||
)
|
||||
|
||||
cost: Final = completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
assert cost == 0.0
|
||||
with pytest.raises(ModelNotMappedError, match="input_cost_per_token"):
|
||||
completion_cost(completion_response=response, model=model, custom_llm_provider="bedrock")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom_llm_provider", ["openai", "hosted_vllm"])
|
||||
|
|
@ -1842,7 +1864,15 @@ def test_cost_calculator_with_cache_creation():
|
|||
}
|
||||
),
|
||||
)
|
||||
model = "claude-sonnet-4@20250514"
|
||||
model: Final = "claude-cache-creation-contract"
|
||||
litellm.register_model({f"vertex_ai/{model}": {
|
||||
"litellm_provider": "vertex_ai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0.000002,
|
||||
"output_cost_per_token": 0.000003,
|
||||
"cache_read_input_token_cost": 0.0000001,
|
||||
"cache_creation_input_token_cost": 0.000004,
|
||||
}})
|
||||
|
||||
assert litellm_model_response.usage.prompt_tokens_details.cached_tokens == 28491
|
||||
|
||||
|
|
@ -1852,7 +1882,7 @@ def test_cost_calculator_with_cache_creation():
|
|||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
print(result)
|
||||
assert result == pytest.approx(13 * 0.000003 + 28491 * 0.0000001 + 15 * 0.000004)
|
||||
|
||||
|
||||
def test_bedrock_cost_calculator_comparison_with_without_cache():
|
||||
|
|
@ -7350,6 +7380,45 @@ def test_vertex_ai_medlm_completion_cost():
|
|||
assert predictive_cost > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("direction", "threshold_field", "prompt_tokens", "completion_tokens", "expected"),
|
||||
[
|
||||
("output", None, 200001, 10, (20.0, 30.0)),
|
||||
("output", "output_cost_per_token_above_200k_tokens", 200001, 10, (20.0, 50.0)),
|
||||
("output", "output_cost_per_token_above_128k_tokens", 200001, 10, (20.0, 30.0)),
|
||||
("output", "output_cost_per_token_above_128k_tokens", 10, 128001, (20.0, 640005.0)),
|
||||
("input", None, 10, 200001, (30.0, 20.0)),
|
||||
("input", "input_cost_per_token_above_200k_tokens", 200001, 10, (1000005.0, 20.0)),
|
||||
("input", "input_cost_per_token_above_128k_tokens", 200001, 10, (1000005.0, 20.0)),
|
||||
],
|
||||
)
|
||||
def test_vertex_mixed_character_pricing_only_prices_the_used_token_direction(
|
||||
direction: Literal["input", "output"],
|
||||
threshold_field: str | None,
|
||||
prompt_tokens: int,
|
||||
completion_tokens: int,
|
||||
expected: tuple[float, float],
|
||||
) -> None:
|
||||
model: Final = "vertex_ai/mixed-character-contract"
|
||||
character_direction: Final = "input" if direction == "output" else "output"
|
||||
litellm.register_model({model: {
|
||||
"litellm_provider": "vertex_ai",
|
||||
"mode": "chat",
|
||||
f"{character_direction}_cost_per_character": 2.0,
|
||||
f"{direction}_cost_per_token": 3.0,
|
||||
**({threshold_field: 5.0} if threshold_field is not None else {}),
|
||||
}})
|
||||
|
||||
assert cost_per_token(
|
||||
model=model,
|
||||
custom_llm_provider="vertex_ai",
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
prompt_characters=10 if character_direction == "input" else None,
|
||||
completion_characters=10 if character_direction == "output" else None,
|
||||
) == expected
|
||||
|
||||
|
||||
def test_vertex_ai_embedding_completion_cost(caplog):
|
||||
"""
|
||||
Relevant issue - https://github.com/BerriAI/litellm/issues/4630
|
||||
|
|
|
|||
|
|
@ -1,26 +1,16 @@
|
|||
"""
|
||||
Test that register_model() in completion() and embedding() passes all
|
||||
custom pricing fields from kwargs and model_info, not just the base
|
||||
input/output costs.
|
||||
|
||||
Previously, only input_cost_per_token, output_cost_per_token, and
|
||||
litellm_provider were forwarded. Fields like cache_read_input_token_cost,
|
||||
mode, and supports_prompt_caching were dropped, causing incorrect cost
|
||||
calculations for DB-sourced models with prompt caching pricing.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import itertools
|
||||
import os
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from queue import Queue
|
||||
from typing import Final, NoReturn
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import pinned_billing_time
|
||||
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map
|
||||
from litellm.main import _build_custom_pricing_entry
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
|
||||
|
|
@ -28,6 +18,350 @@ def _snapshot_model_cost_entries(keys):
|
|||
return {key: copy.deepcopy(litellm.model_cost.get(key)) for key in keys}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
def test_sdk_custom_prices_do_not_leak_between_tenants_or_into_deployments(
|
||||
monkeypatch: pytest.MonkeyPatch, stream: bool
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
|
||||
model: Final = "openai/m5-tenant-backend"
|
||||
deployment_id: Final = "m5-tenant-deployment"
|
||||
catalog_row: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0}
|
||||
owned_row: Final = {"litellm_provider": "openai", "input_cost_per_token": 7.0, "output_cost_per_token": 8.0}
|
||||
monkeypatch.setitem(litellm.model_cost, model, catalog_row.copy())
|
||||
monkeypatch.setitem(litellm.model_cost, deployment_id, owned_row.copy())
|
||||
events: Final[Queue[tuple[object, object]]] = Queue()
|
||||
|
||||
def on_success(kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime) -> None:
|
||||
if kwargs.get("stream") and kwargs.get("complete_streaming_response") is None:
|
||||
return
|
||||
events.put((kwargs.get("response_cost"), kwargs.get("complete_streaming_response") or response))
|
||||
|
||||
monkeypatch.setattr(litellm, "success_callback", [on_success])
|
||||
for input_rate, output_rate, per_call in (
|
||||
(5.0, 6.0, {}),
|
||||
(
|
||||
1.0,
|
||||
4.0,
|
||||
{
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 4.0,
|
||||
"metadata": {"model_info": {"id": deployment_id}},
|
||||
},
|
||||
),
|
||||
(5.0, 6.0, {}),
|
||||
):
|
||||
result: Final = litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
mock_response="Hello back",
|
||||
stream=stream,
|
||||
**per_call,
|
||||
)
|
||||
if isinstance(result, CustomStreamWrapper):
|
||||
assert tuple(result)
|
||||
cost, response = events.get(timeout=10)
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
assert isinstance(response.usage, litellm.Usage)
|
||||
expected: Final = response.usage.prompt_tokens * input_rate + response.usage.completion_tokens * output_rate
|
||||
assert cost == pytest.approx(expected)
|
||||
if not stream:
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(expected)
|
||||
assert litellm.model_cost[model] == catalog_row
|
||||
assert litellm.model_cost[deployment_id] == owned_row
|
||||
assert litellm.cost_per_token(deployment_id, 10, 20, custom_llm_provider="openai") == pytest.approx((70.0, 160.0))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prices", "usage", "service_tier", "expected"),
|
||||
(
|
||||
(
|
||||
{"output_cost_per_reasoning_token": 3.0},
|
||||
litellm.Usage(prompt_tokens=10, completion_tokens=10, completion_tokens_details={"reasoning_tokens": 5}),
|
||||
"default",
|
||||
35.0,
|
||||
),
|
||||
(
|
||||
{"input_cost_per_audio_token": 3.0, "output_cost_per_audio_token": 4.0},
|
||||
litellm.Usage(
|
||||
prompt_tokens=10,
|
||||
completion_tokens=10,
|
||||
prompt_tokens_details={"text_tokens": 6, "audio_tokens": 4},
|
||||
completion_tokens_details={"text_tokens": 6, "audio_tokens": 4},
|
||||
),
|
||||
"default",
|
||||
46.0,
|
||||
),
|
||||
(
|
||||
{"input_cost_per_token_priority": 3.0, "output_cost_per_token_priority": 4.0},
|
||||
litellm.Usage(prompt_tokens=10, completion_tokens=10),
|
||||
"priority",
|
||||
70.0,
|
||||
),
|
||||
),
|
||||
ids=("reasoning", "audio", "priority"),
|
||||
)
|
||||
def test_sdk_request_prices_preserve_non_base_token_fields_without_catalog_writes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
prices: Mapping[str, float],
|
||||
usage: litellm.Usage,
|
||||
service_tier: str,
|
||||
expected: float,
|
||||
) -> None:
|
||||
model: Final = "openai/m5-request-token-fields"
|
||||
row: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0}
|
||||
monkeypatch.setitem(litellm.model_cost, model, row.copy())
|
||||
response: Final = litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=1.0,
|
||||
output_cost_per_token=2.0,
|
||||
mock_response=litellm.ModelResponse(model=model, usage=usage, service_tier=service_tier),
|
||||
**prices,
|
||||
)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(expected)
|
||||
assert litellm.model_cost[model] == row
|
||||
assert litellm.cost_per_token(model, 10, 10, custom_llm_provider="openai") == pytest.approx((50.0, 60.0))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
def test_router_request_prices_override_only_the_current_call(monkeypatch: pytest.MonkeyPatch, stream: bool) -> None:
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
|
||||
model: Final = "openai/m5-routed-request"
|
||||
deployment_id: Final = "m5-routed-request-deployment"
|
||||
catalog: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0}
|
||||
monkeypatch.setitem(litellm.model_cost, model, catalog.copy())
|
||||
events: Final = Queue[tuple[object, object]]()
|
||||
|
||||
class Costs(CustomLogger):
|
||||
def log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
events.put((kwargs.get("response_cost"), kwargs.get("complete_streaming_response", response_obj)))
|
||||
|
||||
monkeypatch.setattr(litellm, "callbacks", [Costs()])
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "m5-request-group",
|
||||
"litellm_params": {"model": model, "api_key": "fake-key"},
|
||||
"model_info": {
|
||||
"id": deployment_id,
|
||||
"input_cost_per_token": 7.0,
|
||||
"output_cost_per_token": 8.0,
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
owned: Final = copy.deepcopy(litellm.model_cost[deployment_id])
|
||||
try:
|
||||
for input_rate, output_rate, per_call in (
|
||||
(7.0, 8.0, {}),
|
||||
(1.0, 2.0, {"input_cost_per_token": 1.0, "output_cost_per_token": 2.0}),
|
||||
(0.0, 0.0, {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0}),
|
||||
(7.0, 8.0, {}),
|
||||
):
|
||||
result: Final = router.completion(
|
||||
model="m5-request-group",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
mock_response="Hello back",
|
||||
stream=stream,
|
||||
**per_call,
|
||||
)
|
||||
if isinstance(result, CustomStreamWrapper):
|
||||
assert tuple(result)
|
||||
cost, response = events.get(timeout=10)
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
assert isinstance(response.usage, litellm.Usage)
|
||||
assert cost == pytest.approx(
|
||||
response.usage.prompt_tokens * input_rate + response.usage.completion_tokens * output_rate
|
||||
)
|
||||
assert litellm.model_cost[deployment_id] == owned
|
||||
assert litellm.model_cost[model] == catalog
|
||||
finally:
|
||||
router.discard()
|
||||
|
||||
|
||||
def test_request_prices_reach_callback_cache_and_reasoning_breakdown_without_registration(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
||||
model: Final = "openai/m5-request-breakdown"
|
||||
row: Final = {"litellm_provider": "openai", "input_cost_per_token": 5.0, "output_cost_per_token": 6.0}
|
||||
monkeypatch.setitem(litellm.model_cost, model, row.copy())
|
||||
events: Final = Queue[object]()
|
||||
|
||||
class Costs(CustomLogger):
|
||||
def log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
|
||||
) -> None:
|
||||
events.put(kwargs.get("standard_logging_object"))
|
||||
|
||||
monkeypatch.setattr(litellm, "callbacks", [Costs()])
|
||||
response: Final = litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=1.0,
|
||||
output_cost_per_token=2.0,
|
||||
cache_read_input_token_cost=0.1,
|
||||
cache_creation_input_token_cost=1.25,
|
||||
output_cost_per_reasoning_token=3.0,
|
||||
model_info={"mode": "chat", "supports_prompt_caching": True, "max_tokens": 8192},
|
||||
mock_response=litellm.ModelResponse(
|
||||
model=model,
|
||||
usage=litellm.Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=10,
|
||||
prompt_tokens_details={"cached_tokens": 60, "cache_creation_tokens": 20},
|
||||
completion_tokens_details={"reasoning_tokens": 5},
|
||||
),
|
||||
),
|
||||
)
|
||||
payload: Final = events.get(timeout=10)
|
||||
assert isinstance(payload, dict)
|
||||
assert payload["response_cost"] == pytest.approx(76.0)
|
||||
assert payload["cost_breakdown"] == pytest.approx(
|
||||
{
|
||||
"input_cost": 51.0,
|
||||
"output_cost": 25.0,
|
||||
"total_cost": 76.0,
|
||||
"tool_usage_cost": 0.0,
|
||||
"service_tier": None,
|
||||
"data_residency": None,
|
||||
"vertex_location": None,
|
||||
"cache_read_cost": 6.0,
|
||||
"cache_creation_cost": 25.0,
|
||||
"reasoning_cost": 15.0,
|
||||
"original_cost": 76.0,
|
||||
"discount_percent": 0.0,
|
||||
"discount_amount": 0.0,
|
||||
"margin_percent": 0.0,
|
||||
"margin_fixed_amount": 0.0,
|
||||
"margin_total_amount": 0.0,
|
||||
}
|
||||
)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(76.0)
|
||||
assert litellm.model_cost[model] == row
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("cache_price", "expected"), ((None, 0.5), (0.0, 0.0), (0.25, 0.25)))
|
||||
def test_request_all_cache_usage_retains_unused_rates_and_zero_overrides(
|
||||
monkeypatch: pytest.MonkeyPatch, cache_price: float | None, expected: float
|
||||
) -> None:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
model: Final = "openai/m5-request-all-cache"
|
||||
row: Final = {
|
||||
"litellm_provider": "openai",
|
||||
"input_cost_per_token": 5.0,
|
||||
"output_cost_per_token": 6.0,
|
||||
"cache_read_input_token_cost": 0.5,
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, model, row.copy())
|
||||
when: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
logging_obj: Final = Logging(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=when,
|
||||
litellm_call_id="m5-request-all-cache",
|
||||
function_id="f",
|
||||
kwargs={
|
||||
"input_cost_per_token": 1.0,
|
||||
"output_cost_per_token": 2.0,
|
||||
"cache_read_input_token_cost": cache_price,
|
||||
},
|
||||
)
|
||||
with pinned_billing_time(when):
|
||||
total: Final = litellm.completion_cost(
|
||||
model=model,
|
||||
custom_llm_provider="openai",
|
||||
custom_pricing=True,
|
||||
litellm_logging_obj=logging_obj,
|
||||
completion_response=litellm.ModelResponse(
|
||||
model=model,
|
||||
usage=litellm.Usage(
|
||||
prompt_tokens=100, completion_tokens=0, prompt_tokens_details={"cached_tokens": 100}
|
||||
),
|
||||
),
|
||||
)
|
||||
assert total == pytest.approx(100 * expected)
|
||||
assert logging_obj.billed_token_rates is not None
|
||||
assert logging_obj.billed_token_rates.input_cost_per_token == 1.0
|
||||
assert logging_obj.billed_token_rates.output_cost_per_token == 2.0
|
||||
assert logging_obj.billed_token_rates.cache_read_input_token_cost == expected
|
||||
assert litellm.model_cost[model] == row
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("speed", "geo", "factor"), (("standard", "global", 1.0), ("fast", "us", 6.6)))
|
||||
def test_ephemeral_request_prices_keep_provider_selected_adjustments(
|
||||
monkeypatch: pytest.MonkeyPatch, speed: str, geo: str, factor: float
|
||||
) -> None:
|
||||
model: Final = "anthropic/m5-request-adjustments"
|
||||
row: Final = {
|
||||
"litellm_provider": "anthropic",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 5.0,
|
||||
"output_cost_per_token": 6.0,
|
||||
"provider_specific_entry": {"fast": 6.0, "us": 1.1},
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, model, copy.deepcopy(row))
|
||||
response: Final = litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=1.0,
|
||||
output_cost_per_token=2.0,
|
||||
mock_response=litellm.ModelResponse(
|
||||
model=model,
|
||||
usage=litellm.Usage(prompt_tokens=10, completion_tokens=10, speed=speed, inference_geo=geo),
|
||||
),
|
||||
)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(30.0 * factor)
|
||||
assert litellm.model_cost[model] == row
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("input_cost_per_second", "output_cost_per_second", "cost_per_second"))
|
||||
@pytest.mark.parametrize("rate", (0.0, 0.02))
|
||||
def test_request_duration_prices_keep_python_time_billing_without_registration(field: str, rate: float) -> None:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
|
||||
model: Final = "openai/m5-request-duration"
|
||||
logging_obj: Final = Logging(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
stream=False,
|
||||
call_type="completion",
|
||||
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
litellm_call_id="m5-request-duration",
|
||||
function_id="f",
|
||||
kwargs={field: rate},
|
||||
)
|
||||
total: Final = litellm.completion_cost(
|
||||
model=model,
|
||||
custom_llm_provider="openai",
|
||||
custom_pricing=True,
|
||||
litellm_logging_obj=logging_obj,
|
||||
total_time=1500.0,
|
||||
completion_response=litellm.ModelResponse(
|
||||
model=model, usage=litellm.Usage(prompt_tokens=10, completion_tokens=10)
|
||||
),
|
||||
)
|
||||
assert total == pytest.approx(rate * 1.5)
|
||||
assert logging_obj.cost_breakdown is not None
|
||||
assert logging_obj.cost_breakdown["input_cost"] == pytest.approx(rate * 1.5)
|
||||
assert logging_obj.cost_breakdown["output_cost"] == 0.0
|
||||
assert model not in litellm.model_cost
|
||||
|
||||
|
||||
def _restore_model_cost_entries(original_entries):
|
||||
for key, value in original_entries.items():
|
||||
if value is None:
|
||||
|
|
@ -37,131 +371,6 @@ def _restore_model_cost_entries(original_entries):
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_includes_all_kwargs_fields():
|
||||
"""All CustomPricingLiteLLMParams fields present in kwargs should be
|
||||
included in the resulting entry dict."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
"cache_read_input_token_cost": 0.00025,
|
||||
"cache_creation_input_token_cost": 0.005,
|
||||
"output_cost_per_reasoning_token": 0.01,
|
||||
"input_cost_per_audio_token": 0.003,
|
||||
"unrelated_kwarg": "should_be_ignored",
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert entry["litellm_provider"] == "openai"
|
||||
assert entry["input_cost_per_token"] == 0.001
|
||||
assert entry["output_cost_per_token"] == 0.002
|
||||
assert entry["cache_read_input_token_cost"] == 0.00025
|
||||
assert entry["cache_creation_input_token_cost"] == 0.005
|
||||
assert entry["output_cost_per_reasoning_token"] == 0.01
|
||||
assert entry["input_cost_per_audio_token"] == 0.003
|
||||
assert "unrelated_kwarg" not in entry
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_merges_model_info_metadata():
|
||||
"""Fields from model_info (mode, supports_prompt_caching, max_tokens)
|
||||
should be merged into the entry when present."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
}
|
||||
model_info = {
|
||||
"id": "deployment-123",
|
||||
"mode": "chat",
|
||||
"supports_prompt_caching": True,
|
||||
"max_tokens": 128000,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert entry["mode"] == "chat"
|
||||
assert entry["supports_prompt_caching"] is True
|
||||
assert entry["max_tokens"] == 128000
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_setdefault_does_not_override_existing():
|
||||
"""model_info uses setdefault, so it should not override a key that is
|
||||
already present in the entry dict. Currently CustomPricingLiteLLMParams
|
||||
and the model_info keys (mode, supports_prompt_caching, max_tokens) do
|
||||
not overlap, but if they ever do, setdefault ensures the kwargs-sourced
|
||||
value wins."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
}
|
||||
model_info = {
|
||||
"mode": "chat",
|
||||
"supports_prompt_caching": True,
|
||||
"max_tokens": 128000,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert entry["mode"] == "chat"
|
||||
assert entry["supports_prompt_caching"] is True
|
||||
assert entry["max_tokens"] == 128000
|
||||
|
||||
# Verify setdefault behavior: if a model_info key already exists in
|
||||
# the entry (e.g. from a future CustomPricingLiteLLMParams addition),
|
||||
# setdefault must not overwrite it.
|
||||
entry["mode"] = "embedding" # simulate pre-existing value
|
||||
# Re-apply setdefault the same way _build_custom_pricing_entry does
|
||||
entry.setdefault("mode", model_info["mode"])
|
||||
assert entry["mode"] == "embedding" # must NOT revert to "chat"
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_skips_none_values():
|
||||
"""Fields with None values in kwargs should not be included."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": None, # explicitly None
|
||||
"cache_read_input_token_cost": None,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert entry["input_cost_per_token"] == 0.001
|
||||
assert "output_cost_per_token" not in entry
|
||||
assert "cache_read_input_token_cost" not in entry
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_handles_no_model_info():
|
||||
"""Should work correctly when model_info is None."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
model_info=None,
|
||||
)
|
||||
|
||||
assert entry["litellm_provider"] == "openai"
|
||||
assert entry["input_cost_per_token"] == 0.001
|
||||
assert entry["output_cost_per_token"] == 0.002
|
||||
assert "mode" not in entry
|
||||
|
||||
|
||||
def test_register_model_receives_cache_pricing_fields():
|
||||
"""End-to-end: when register_model is called with a full pricing entry,
|
||||
the cache pricing fields should be present in litellm.model_cost."""
|
||||
|
|
@ -192,23 +401,6 @@ def test_register_model_receives_cache_pricing_fields():
|
|||
litellm.model_cost.pop(model_key, None)
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_time_based():
|
||||
"""Time-based pricing fields should be included correctly."""
|
||||
kwargs = {
|
||||
"input_cost_per_second": 0.01,
|
||||
"output_cost_per_second": 0.02,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert entry["litellm_provider"] == "openai"
|
||||
assert entry["input_cost_per_second"] == 0.01
|
||||
assert entry["output_cost_per_second"] == 0.02
|
||||
|
||||
|
||||
def test_register_model_strips_none_litellm_provider():
|
||||
"""``get_model_info`` returns ``litellm_provider: None`` for deployments
|
||||
registered without a provider (e.g. ``Router.add_deployment`` flows).
|
||||
|
|
@ -598,6 +790,16 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
assert builtin_input_cost > 0
|
||||
|
||||
try:
|
||||
litellm.register_model(
|
||||
{
|
||||
deployment_id: {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "embedding",
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
}
|
||||
}
|
||||
)
|
||||
litellm.embedding(
|
||||
model=shared_key,
|
||||
input=["hello"],
|
||||
|
|
@ -640,6 +842,16 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
|
|||
assert builtin_input_cost != override_input_cost
|
||||
|
||||
try:
|
||||
litellm.register_model(
|
||||
{
|
||||
deployment_id: {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "embedding",
|
||||
"input_cost_per_token": override_input_cost,
|
||||
"output_cost_per_token": override_input_cost * 2,
|
||||
}
|
||||
}
|
||||
)
|
||||
response = litellm.embedding(
|
||||
model=shared_key,
|
||||
input=["hello"],
|
||||
|
|
@ -674,6 +886,16 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
assert builtin_input_cost > 0
|
||||
|
||||
try:
|
||||
litellm.register_model(
|
||||
{
|
||||
deployment_id: {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
}
|
||||
}
|
||||
)
|
||||
litellm.completion(
|
||||
model=shared_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
|
|
@ -693,11 +915,7 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
_restore_model_cost_entries(snapshot)
|
||||
|
||||
|
||||
def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key():
|
||||
"""Direct SDK calls (no router deployment id in metadata) keep the legacy
|
||||
behavior: custom pricing is registered under ``{provider}/{model}`` and the
|
||||
request is costed with it.
|
||||
"""
|
||||
def test_embedding_direct_sdk_custom_pricing_does_not_register_shared_key():
|
||||
model_key = "openai/lit3991-direct-sdk-embed-model"
|
||||
override_input_cost = 3e-05
|
||||
try:
|
||||
|
|
@ -710,13 +928,18 @@ def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key():
|
|||
mock_response=[0.1, 0.2],
|
||||
)
|
||||
|
||||
assert litellm.model_cost[model_key]["input_cost_per_token"] == override_input_cost
|
||||
assert model_key not in litellm.model_cost
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(10 * override_input_cost)
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=model_key,
|
||||
custom_llm_provider="openai",
|
||||
call_type="embedding",
|
||||
custom_pricing=True,
|
||||
custom_cost_per_token={
|
||||
"input_cost_per_token": override_input_cost,
|
||||
"output_cost_per_token": override_input_cost * 2,
|
||||
},
|
||||
)
|
||||
assert cost == pytest.approx(10 * override_input_cost)
|
||||
finally:
|
||||
|
|
@ -1417,10 +1640,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing():
|
|||
object stays untouched, and the caller's incoming nested dict is never
|
||||
inserted by reference into the merged result.
|
||||
"""
|
||||
from litellm.utils import update_dictionary
|
||||
|
||||
from typing import cast
|
||||
|
||||
from litellm.utils import update_dictionary
|
||||
|
||||
existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"})
|
||||
existing = {"off_peak_pricing": existing_nested}
|
||||
incoming_nested = cast(
|
||||
|
|
@ -1619,12 +1842,12 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing():
|
|||
del router
|
||||
|
||||
|
||||
def test_completion_registers_cost_per_second_pricing():
|
||||
def test_completion_uses_ephemeral_cost_per_second_pricing():
|
||||
model_key: Final = "openai/test-cost-per-second-registration"
|
||||
original_entries: Final = _snapshot_model_cost_entries([model_key])
|
||||
|
||||
try:
|
||||
litellm.completion(
|
||||
response: Final = litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
|
|
@ -1632,7 +1855,14 @@ def test_completion_registers_cost_per_second_pricing():
|
|||
mock_response="hello back",
|
||||
)
|
||||
|
||||
assert litellm.model_cost[model_key]["cost_per_second"] == 0.02
|
||||
assert model_key not in litellm.model_cost
|
||||
assert response._hidden_params["response_cost"] is not None
|
||||
assert litellm.completion_cost(
|
||||
litellm.ModelResponse(model=model_key),
|
||||
model=model_key,
|
||||
custom_cost_per_second=0.02,
|
||||
total_time=2000,
|
||||
) == pytest.approx(0.04)
|
||||
finally:
|
||||
_restore_model_cost_entries(original_entries)
|
||||
|
||||
|
|
@ -1686,9 +1916,7 @@ def test_register_model_scopes_builtin_match_to_the_given_provider():
|
|||
_restore_model_cost_entries(model_cost_entries)
|
||||
|
||||
|
||||
def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider():
|
||||
from litellm.main import _register_custom_pricing_for_request
|
||||
|
||||
def test_per_request_custom_pricing_does_not_write_a_colliding_deployment_id():
|
||||
deployment_id: Final = "baseten/zai-org/glm-5.2"
|
||||
builtin_key: Final = "baseten/zai-org/GLM-5.2"
|
||||
shared_key: Final = "openai/zai-org/GLM-5.2"
|
||||
|
|
@ -1696,20 +1924,20 @@ def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_prov
|
|||
builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key])
|
||||
openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models)
|
||||
try:
|
||||
_register_custom_pricing_for_request(
|
||||
model="zai-org/GLM-5.2",
|
||||
custom_llm_provider="openai",
|
||||
kwargs={
|
||||
"input_cost_per_token": 0.00000096,
|
||||
"output_cost_per_token": 0.00000302,
|
||||
"metadata": {"model_info": {"id": deployment_id}},
|
||||
},
|
||||
model_info={"mode": "chat"},
|
||||
response: Final = litellm.completion(
|
||||
model=shared_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
input_cost_per_token=0.00000096,
|
||||
output_cost_per_token=0.00000302,
|
||||
metadata={"model_info": {"id": deployment_id}},
|
||||
mock_response=litellm.ModelResponse(
|
||||
model=shared_key, usage=litellm.Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30)
|
||||
),
|
||||
)
|
||||
|
||||
entry: Final = litellm.model_cost[deployment_id]
|
||||
assert entry["input_cost_per_token"] == 0.00000096
|
||||
assert entry["output_cost_per_token"] == 0.00000302
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(10 * 0.00000096 + 20 * 0.00000302)
|
||||
assert litellm.model_cost.get(deployment_id) == model_cost_entries[deployment_id]
|
||||
assert litellm.model_cost[builtin_key] == builtin_row_before
|
||||
finally:
|
||||
_restore_model_cost_entries(model_cost_entries)
|
||||
|
|
|
|||
|
|
@ -3161,12 +3161,10 @@ def test_inherit_builtin_tiered_output_rate_fills_the_backend_flat_rate():
|
|||
|
||||
def test_inherit_builtin_tiered_output_rate_never_stores_a_synthesized_zero():
|
||||
"""
|
||||
Regression: get_model_info reports output_cost_per_token 0 for a backend that
|
||||
only publishes tiered rates (e.g. dashscope/qwen-flash), and storing that zero
|
||||
would mark the deployment as explicitly priced free.
|
||||
Missing flat prices must not mark a tier-priced backend as explicitly free.
|
||||
"""
|
||||
backend_info = litellm.get_model_info(model="qwen-flash", custom_llm_provider="dashscope")
|
||||
assert backend_info["output_cost_per_token"] == 0
|
||||
assert backend_info["output_cost_per_token"] is None
|
||||
|
||||
model_info = {"tiered_pricing": [{"range": [0, 3000], "input_cost_per_token": 3.25e-07}]}
|
||||
Router._inherit_builtin_tiered_output_rate(
|
||||
|
|
|
|||
|
|
@ -2445,7 +2445,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None:
|
|||
|
||||
@respx.mock
|
||||
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
|
||||
def test_completion_registers_test_owned_model_prices() -> None:
|
||||
def test_completion_uses_request_prices_without_registering_them() -> None:
|
||||
model: Final = "migration-owned-pricing-model"
|
||||
messages: Final = [{"role": "user", "content": "price control"}]
|
||||
litellm.register_model(
|
||||
|
|
@ -2488,8 +2488,7 @@ def test_completion_registers_test_owned_model_prices() -> None:
|
|||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages}
|
||||
assert litellm.model_cost[f"openai/{model}"]["input_cost_per_token"] == 0.001
|
||||
assert litellm.model_cost[f"openai/{model}"]["output_cost_per_token"] == 0.002
|
||||
assert f"openai/{model}" not in litellm.model_cost
|
||||
assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001
|
||||
assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(0.018)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue