mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-04 07:27:51 +00:00
fix(main)!: carry the full per-call price entry on the logging object instead of writing litellm.model_cost (#49532)
* fix(main): price per-request custom pricing from the logging object instead of writing litellm.model_cost * fix(main)!: carry the full per-call price entry on the logging object instead of writing litellm.model_cost Direct SDK calls build one ModelInfo from the cost map row plus every custom pricing field on the call and price from it, so audio, tiered and directional per-second rates apply without touching litellm.model_cost. Router calls keep their registered deployment entry. completion_cost(completion_response=...) alone no longer sees per-call prices, pass custom_cost_per_second or custom_cost_per_token * test(cost): assert the exact per-second charge for per-call cost_per_second * fix(main): price Router request overrides and flat rates over inherited tiers in the per-call entry A Router call that sets its own price is priced from it over the registered deployment entry, PTU zeroing applied as at registration. Flat token rates on a call drop an inherited tiered_pricing table, which generic_cost_per_token would otherwise prefer * fix(main): keep tier rates for directions a call does not override and bill per-second-only calls per second A flat rate set on a call is written into every tier for that direction instead of dropping the tier table, so the other direction keeps its tier price. A call that sets only a per-second rate on a model with catalog token rates clears those token rates for the call so the per-second path applies * fix(main): apply a call's flat rate to its own tiered_pricing table when it supplies one * refactor(main): move per-call price entry resolution into litellm_core_utils/per_call_pricing The resolution logic is pure cost-map work with no dependency on dispatch, so it lives next to ptu_pricing and main.py keeps only the call sites in completion() and embedding() Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * refactor(cost): rename per_call_model_info parameter to requested_model_info_override Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kerry Lu
GitHub
parent
2e0d333de6
commit
65950dba8b
7 changed files with 448 additions and 433 deletions
|
|
@ -442,6 +442,14 @@ def _per_second_pricing_cost(
|
|||
model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
|
||||
return None
|
||||
return _per_second_pricing_cost_from_info(model_info, response_time_ms, audio_seconds)
|
||||
|
||||
|
||||
def _per_second_pricing_cost_from_info(
|
||||
model_info: ModelInfoBase,
|
||||
response_time_ms: float | None,
|
||||
audio_seconds: float = 0.0,
|
||||
) -> tuple[float, float] | None:
|
||||
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
|
||||
return None
|
||||
cost_per_second: Final = model_info.get("cost_per_second")
|
||||
|
|
@ -464,7 +472,7 @@ def _per_second_pricing_cost(
|
|||
)
|
||||
verbose_logger.debug(
|
||||
"For model=%s - cost_per_second: %s; response time: %s",
|
||||
model,
|
||||
model_info.get("key"),
|
||||
resolved_cost_per_second,
|
||||
response_time_ms,
|
||||
)
|
||||
|
|
@ -487,6 +495,7 @@ def cost_per_token(
|
|||
### CUSTOM PRICING ###
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
requested_model_info_override: ModelInfo | None = None,
|
||||
### NUMBER OF QUERIES ###
|
||||
number_of_queries: int | None = None,
|
||||
### USAGE OBJECT ###
|
||||
|
|
@ -598,6 +607,20 @@ def cost_per_token(
|
|||
if response_cost is not None:
|
||||
return response_cost[0], response_cost[1]
|
||||
|
||||
if requested_model_info_override is not None:
|
||||
per_second: Final = _per_second_pricing_cost_from_info(
|
||||
requested_model_info_override, response_time_ms, audio_transcription_file_duration
|
||||
)
|
||||
if per_second is not None:
|
||||
return per_second
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider or "",
|
||||
service_tier=service_tier,
|
||||
model_info=requested_model_info_override,
|
||||
)
|
||||
|
||||
model_cost_ref: Final = litellm.model_cost
|
||||
# Only callers that explicitly pass `custom_llm_provider` get the
|
||||
# dedup/prefix-join treatment. When provider is omitted, preserve legacy
|
||||
|
|
@ -1747,6 +1770,7 @@ def completion_cost(
|
|||
### CUSTOM PRICING ###
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
requested_model_info_override: ModelInfo | None = None,
|
||||
optional_params: dict | None = None,
|
||||
custom_pricing: bool | None = None,
|
||||
base_model: str | None = None,
|
||||
|
|
@ -1800,6 +1824,7 @@ def completion_cost(
|
|||
quality=quality,
|
||||
n=n,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
requested_model_info_override=requested_model_info_override,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
|
|
@ -2274,6 +2299,7 @@ def completion_cost(
|
|||
region_name=None if explicit_pricing else region_name,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
requested_model_info_override=requested_model_info_override,
|
||||
prompt_characters=prompt_characters,
|
||||
completion_characters=completion_characters,
|
||||
cache_creation_input_tokens=cache_creation_input_tokens,
|
||||
|
|
@ -2508,6 +2534,7 @@ def response_cost_calculator(
|
|||
region_name: str | None = None,
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
requested_model_info_override: ModelInfo | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Returns
|
||||
|
|
@ -2564,6 +2591,7 @@ def response_cost_calculator(
|
|||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
requested_model_info_override=requested_model_info_override,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
|
|
|
|||
|
|
@ -722,6 +722,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
custom_pricing: bool = False
|
||||
custom_cost_per_token: CostPerToken | None = None
|
||||
custom_cost_per_second: float | None = None
|
||||
custom_model_info: ModelInfo | None = None
|
||||
replaced_global_callback_keys: frozenset[str] = frozenset()
|
||||
stream_options = None
|
||||
litellm_request_debug: bool = False
|
||||
|
|
@ -2227,6 +2228,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"custom_pricing": custom_pricing,
|
||||
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
|
||||
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
|
||||
"requested_model_info_override": self.custom_model_info if custom_pricing else None,
|
||||
"prompt": prompt,
|
||||
"standard_built_in_tools_params": self.standard_built_in_tools_params,
|
||||
"router_model_id": router_model_id,
|
||||
|
|
@ -2423,6 +2425,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
additional_headers=additional_headers,
|
||||
)
|
||||
|
||||
def set_per_call_pricing(self, custom_model_info: ModelInfo | None) -> None:
|
||||
"""Price this call from its own full price entry instead of ``litellm.model_cost``."""
|
||||
self.custom_model_info = custom_model_info
|
||||
|
||||
def custom_pricing_for(self, result: object) -> bool:
|
||||
if self.spilled_over_ptu_for(result):
|
||||
return False
|
||||
|
|
|
|||
104
litellm/litellm_core_utils/per_call_pricing.py
Normal file
104
litellm/litellm_core_utils/per_call_pricing.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
"""The price entry for a single call that sets its own custom pricing, built without writing ``litellm.model_cost``."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.ptu_pricing import zeroed_ptu_pricing
|
||||
from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfo
|
||||
|
||||
_STR_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODEL_INFO: Final = TypeAdapter(ModelInfo)
|
||||
|
||||
|
||||
def _router_deployment_model_info(kwargs: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
"""A Router stamps the deployment's ``model_info`` (with its ``id``) into the call's metadata and registers that
|
||||
deployment's full custom pricing under that id when the deployment is added."""
|
||||
for metadata_key in ("litellm_metadata", "metadata"):
|
||||
metadata = kwargs.get(metadata_key)
|
||||
if not isinstance(metadata, Mapping):
|
||||
continue
|
||||
model_info = _STR_MAPPING.validate_python(metadata).get("model_info")
|
||||
if isinstance(model_info, Mapping) and isinstance(model_info.get("id"), str):
|
||||
return _STR_MAPPING.validate_python(model_info)
|
||||
return None
|
||||
|
||||
|
||||
def per_call_model_info(
|
||||
model: str, custom_llm_provider: str, kwargs: Mapping[str, object], mode: Literal["chat", "embedding"]
|
||||
) -> ModelInfo | None:
|
||||
"""The full price entry for one call, or None when the cost map (or the Router's registered deployment entry)
|
||||
already prices it. The cost map row for the model is overlaid with the registered deployment entry and then with
|
||||
the custom pricing fields the request itself set, so a Router call pays request overrides over deployment prices.
|
||||
The entry travels on the logging object and is never written into ``litellm.model_cost``, so the next call for the
|
||||
same model name is priced from the cost map again."""
|
||||
deployment: Final = _router_deployment_model_info(kwargs)
|
||||
registered: Final = litellm.model_cost.get(str(deployment["id"])) if deployment is not None else None
|
||||
custom: Final = {
|
||||
field: kwargs[field]
|
||||
for field in CustomPricingLiteLLMParams.model_fields
|
||||
if kwargs.get(field) is not None and (registered is None or registered.get(field) != kwargs[field])
|
||||
}
|
||||
if not custom:
|
||||
return None
|
||||
unmapped: Final[Mapping[str, object]] = {
|
||||
"key": model,
|
||||
"max_tokens": None,
|
||||
"max_input_tokens": None,
|
||||
"max_output_tokens": None,
|
||||
"input_cost_per_token": None,
|
||||
"output_cost_per_token": None,
|
||||
"litellm_provider": custom_llm_provider,
|
||||
"mode": mode,
|
||||
"supported_openai_params": None,
|
||||
}
|
||||
row: Final = _catalog_row(model, custom_llm_provider) or unmapped
|
||||
catalog: Final = (
|
||||
{**row, **{key: value for key, value in registered.items() if value is not None}} if registered else row
|
||||
)
|
||||
zeroed: Final = zeroed_ptu_pricing(deployment, custom) if deployment is not None else None
|
||||
return _MODEL_INFO.validate_python(
|
||||
{
|
||||
**catalog,
|
||||
**_token_rates_cleared_for_per_second_overrides(custom),
|
||||
**custom,
|
||||
**_tiers_with_flat_overrides(catalog, custom),
|
||||
**(zeroed or {}),
|
||||
"mode": catalog.get("mode") or mode,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_FLAT_TOKEN_RATE_FIELDS: Final = ("input_cost_per_token", "output_cost_per_token")
|
||||
_PER_SECOND_RATE_FIELDS: Final = ("cost_per_second", "input_cost_per_second", "output_cost_per_second")
|
||||
_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
|
||||
|
||||
def _token_rates_cleared_for_per_second_overrides(custom: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""Token rates in the entry make the cost calculator skip per-second pricing, so a caller who sets only a
|
||||
per-second rate gets the catalog's token rates and tier table cleared for this call."""
|
||||
if not any(key in custom for key in _PER_SECOND_RATE_FIELDS) or any(
|
||||
key in custom for key in _FLAT_TOKEN_RATE_FIELDS
|
||||
):
|
||||
return {}
|
||||
return {"input_cost_per_token": None, "output_cost_per_token": None, "tiered_pricing": None}
|
||||
|
||||
|
||||
def _tiers_with_flat_overrides(catalog: Mapping[str, object], custom: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""A ``tiered_pricing`` table wins over flat rates in the cost calculator, so a flat rate the caller set is written
|
||||
into every tier of the table in force (the caller's own, else the inherited one) for that direction, and the other
|
||||
direction keeps each tier's own rate."""
|
||||
overrides: Final = {key: custom[key] for key in _FLAT_TOKEN_RATE_FIELDS if key in custom}
|
||||
tiers: Final = custom.get("tiered_pricing", catalog.get("tiered_pricing"))
|
||||
if not overrides or not isinstance(tiers, (list, tuple)) or not tiers:
|
||||
return {}
|
||||
return {"tiered_pricing": [{**tier, **overrides} for tier in _TIERS.validate_python(tiers)]}
|
||||
|
||||
|
||||
def _catalog_row(model: str, custom_llm_provider: str) -> ModelInfo | None:
|
||||
try:
|
||||
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model
|
||||
return None
|
||||
127
litellm/main.py
127
litellm/main.py
|
|
@ -111,6 +111,7 @@ from litellm.litellm_core_utils.mock_functions import (
|
|||
mock_embedding,
|
||||
mock_image_generation,
|
||||
)
|
||||
from litellm.litellm_core_utils.per_call_pricing import per_call_model_info
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_content_from_model_response as get_content_from_model_response, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
|
|
@ -167,7 +168,6 @@ from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_s
|
|||
from litellm.types.utils import (
|
||||
AgenticLoopParams,
|
||||
BaseUrlRoutingHint,
|
||||
CustomPricingLiteLLMParams,
|
||||
ModelResponseStream,
|
||||
ProviderRoutingHints,
|
||||
RawRequestTypedDict,
|
||||
|
|
@ -1323,7 +1323,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o
|
|||
)
|
||||
from litellm.responses.bridge_check import (
|
||||
bridge_reasoning_effort,
|
||||
keep_registered_mode,
|
||||
str_keyed_mapping,
|
||||
)
|
||||
from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site
|
||||
|
|
@ -1429,46 +1428,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]:
|
|||
return values
|
||||
|
||||
|
||||
def _build_custom_pricing_entry(
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
model_info: dict | None = None,
|
||||
) -> dict:
|
||||
"""Build a complete model cost entry from kwargs and model_info.
|
||||
|
||||
Collects all CustomPricingLiteLLMParams fields present in kwargs and
|
||||
merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
|
||||
so that register_model() receives the full pricing configuration.
|
||||
"""
|
||||
entry: Final[dict] = {"litellm_provider": custom_llm_provider}
|
||||
|
||||
for field_name in CustomPricingLiteLLMParams.model_fields:
|
||||
value = kwargs.get(field_name)
|
||||
if value is not None:
|
||||
entry[field_name] = value
|
||||
|
||||
if model_info and isinstance(model_info, dict):
|
||||
for key in ("mode", "supports_prompt_caching", "max_tokens"):
|
||||
if key in model_info and model_info[key] is not None:
|
||||
entry.setdefault(key, model_info[key])
|
||||
|
||||
return entry
|
||||
|
||||
|
||||
def _get_router_deployment_id(kwargs: dict) -> str | None:
|
||||
for metadata_key in ("litellm_metadata", "metadata"):
|
||||
metadata = kwargs.get(metadata_key) or {}
|
||||
if not isinstance(metadata, dict):
|
||||
continue
|
||||
deployment_model_info = metadata.get("model_info") or {}
|
||||
if not isinstance(deployment_model_info, dict):
|
||||
continue
|
||||
deployment_id = deployment_model_info.get("id")
|
||||
if deployment_id is not None:
|
||||
return str(deployment_id)
|
||||
return None
|
||||
|
||||
|
||||
def _record_agentic_loop_params(
|
||||
logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool
|
||||
) -> None:
|
||||
|
|
@ -1479,53 +1438,6 @@ def _record_agentic_loop_params(
|
|||
)
|
||||
|
||||
|
||||
def _register_custom_pricing_for_request(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
model_info: dict | None,
|
||||
) -> None:
|
||||
"""Register per-request custom pricing in litellm.model_cost.
|
||||
|
||||
Router-originated requests (identified by the deployment id the router puts
|
||||
in metadata) get their full pricing registered under that unique id only;
|
||||
the shared ``{provider}/{model}`` key receives the entry with pricing fields
|
||||
stripped, mirroring Router._create_deployment. This keeps one deployment's
|
||||
pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in
|
||||
pricing used by sibling deployments of the same backend model. Direct SDK
|
||||
calls keep the legacy behavior of registering the shared key with pricing.
|
||||
Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on
|
||||
deployment registration, since a call routes by its own ``model_info``.
|
||||
"""
|
||||
entry: Final = _build_custom_pricing_entry(
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
shared_key: Final = f"{custom_llm_provider}/{model}"
|
||||
deployment_id: Final = _get_router_deployment_id(kwargs)
|
||||
if deployment_id is None:
|
||||
litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False)
|
||||
return
|
||||
shared_entry: Final = keep_registered_mode(
|
||||
shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)
|
||||
)
|
||||
litellm.register_model(
|
||||
{deployment_id: entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
litellm.register_model(
|
||||
{shared_key: shared_entry},
|
||||
persist_across_reloads=False,
|
||||
warning_display_name=shared_key,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
add_to_provider_model_sets=False,
|
||||
)
|
||||
|
||||
|
||||
def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None:
|
||||
return ctx.metadata
|
||||
|
||||
|
|
@ -5787,9 +5699,9 @@ def completion(
|
|||
### CUSTOM MODEL COST ###
|
||||
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
|
||||
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
|
||||
cost_per_second: Final = kwargs.get("cost_per_second", None)
|
||||
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
|
||||
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
|
||||
cost_per_second: Final = kwargs.get("cost_per_second", None)
|
||||
### CUSTOM PROMPT TEMPLATE ###
|
||||
initial_prompt_value: Final = kwargs.get("initial_prompt_value", None)
|
||||
roles: Final = kwargs.get("roles", None)
|
||||
|
|
@ -5987,19 +5899,7 @@ def completion(
|
|||
"aws_region_name", None
|
||||
) # support region-based pricing for bedrock
|
||||
|
||||
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
|
||||
if (
|
||||
(input_cost_per_token is not None and output_cost_per_token is not None)
|
||||
or input_cost_per_second is not None
|
||||
or output_cost_per_second is not None
|
||||
or cost_per_second is not None
|
||||
):
|
||||
_register_custom_pricing_for_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
logging.set_per_call_pricing(per_call_model_info(model, custom_llm_provider, kwargs, "chat"))
|
||||
### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ###
|
||||
custom_prompt_dict = {}
|
||||
if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token:
|
||||
|
|
@ -7006,12 +6906,6 @@ def embedding(
|
|||
headers.update(proxy_headers)
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Failed to get proxy auth headers: %s", e)
|
||||
### CUSTOM MODEL COST ###
|
||||
input_cost_per_token: Final = kwargs.get("input_cost_per_token", None)
|
||||
output_cost_per_token: Final = kwargs.get("output_cost_per_token", None)
|
||||
cost_per_second: Final = kwargs.get("cost_per_second", None)
|
||||
input_cost_per_second: Final = kwargs.get("input_cost_per_second", None)
|
||||
output_cost_per_second: Final = kwargs.get("output_cost_per_second", None)
|
||||
openai_params: Final = [
|
||||
"user",
|
||||
"dimensions",
|
||||
|
|
@ -7058,21 +6952,8 @@ def embedding(
|
|||
**non_default_params,
|
||||
)
|
||||
|
||||
### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ###
|
||||
if (
|
||||
(input_cost_per_token is not None and output_cost_per_token is not None)
|
||||
or input_cost_per_second is not None
|
||||
or output_cost_per_second is not None
|
||||
or cost_per_second is not None
|
||||
):
|
||||
_register_custom_pricing_for_request(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
kwargs=kwargs,
|
||||
model_info=kwargs.get("model_info"),
|
||||
)
|
||||
|
||||
logging: Final[LiteLLMLoggingObj] = litellm_logging_obj
|
||||
logging.set_per_call_pricing(per_call_model_info(model, custom_llm_provider, kwargs, "embedding"))
|
||||
logging.update_environment_variables(
|
||||
model=model,
|
||||
user=user,
|
||||
|
|
|
|||
|
|
@ -549,7 +549,7 @@ async def test_completion_sagemaker(
|
|||
|
||||
assert route.called
|
||||
assert response.choices[0].message.content == "SageMaker reply"
|
||||
assert litellm.completion_cost(completion_response=response) == pytest.approx(rate)
|
||||
assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -17,10 +17,16 @@ from typing import Final, NoReturn
|
|||
|
||||
import pytest
|
||||
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map
|
||||
from litellm.main import _build_custom_pricing_entry
|
||||
from litellm.types.utils import (
|
||||
Choices,
|
||||
CompletionTokensDetailsWrapper,
|
||||
Message,
|
||||
ModelResponse,
|
||||
PromptTokensDetailsWrapper,
|
||||
Usage,
|
||||
)
|
||||
from litellm.utils import _invalidate_model_cost_lowercase_map
|
||||
|
||||
|
||||
|
|
@ -37,131 +43,6 @@ def _restore_model_cost_entries(original_entries):
|
|||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_includes_all_kwargs_fields():
|
||||
"""All CustomPricingLiteLLMParams fields present in kwargs should be
|
||||
included in the resulting entry dict."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
"cache_read_input_token_cost": 0.00025,
|
||||
"cache_creation_input_token_cost": 0.005,
|
||||
"output_cost_per_reasoning_token": 0.01,
|
||||
"input_cost_per_audio_token": 0.003,
|
||||
"unrelated_kwarg": "should_be_ignored",
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert entry["litellm_provider"] == "openai"
|
||||
assert entry["input_cost_per_token"] == 0.001
|
||||
assert entry["output_cost_per_token"] == 0.002
|
||||
assert entry["cache_read_input_token_cost"] == 0.00025
|
||||
assert entry["cache_creation_input_token_cost"] == 0.005
|
||||
assert entry["output_cost_per_reasoning_token"] == 0.01
|
||||
assert entry["input_cost_per_audio_token"] == 0.003
|
||||
assert "unrelated_kwarg" not in entry
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_merges_model_info_metadata():
|
||||
"""Fields from model_info (mode, supports_prompt_caching, max_tokens)
|
||||
should be merged into the entry when present."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
}
|
||||
model_info = {
|
||||
"id": "deployment-123",
|
||||
"mode": "chat",
|
||||
"supports_prompt_caching": True,
|
||||
"max_tokens": 128000,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert entry["mode"] == "chat"
|
||||
assert entry["supports_prompt_caching"] is True
|
||||
assert entry["max_tokens"] == 128000
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_setdefault_does_not_override_existing():
|
||||
"""model_info uses setdefault, so it should not override a key that is
|
||||
already present in the entry dict. Currently CustomPricingLiteLLMParams
|
||||
and the model_info keys (mode, supports_prompt_caching, max_tokens) do
|
||||
not overlap, but if they ever do, setdefault ensures the kwargs-sourced
|
||||
value wins."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
}
|
||||
model_info = {
|
||||
"mode": "chat",
|
||||
"supports_prompt_caching": True,
|
||||
"max_tokens": 128000,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
model_info=model_info,
|
||||
)
|
||||
|
||||
assert entry["mode"] == "chat"
|
||||
assert entry["supports_prompt_caching"] is True
|
||||
assert entry["max_tokens"] == 128000
|
||||
|
||||
# Verify setdefault behavior: if a model_info key already exists in
|
||||
# the entry (e.g. from a future CustomPricingLiteLLMParams addition),
|
||||
# setdefault must not overwrite it.
|
||||
entry["mode"] = "embedding" # simulate pre-existing value
|
||||
# Re-apply setdefault the same way _build_custom_pricing_entry does
|
||||
entry.setdefault("mode", model_info["mode"])
|
||||
assert entry["mode"] == "embedding" # must NOT revert to "chat"
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_skips_none_values():
|
||||
"""Fields with None values in kwargs should not be included."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": None, # explicitly None
|
||||
"cache_read_input_token_cost": None,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert entry["input_cost_per_token"] == 0.001
|
||||
assert "output_cost_per_token" not in entry
|
||||
assert "cache_read_input_token_cost" not in entry
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_handles_no_model_info():
|
||||
"""Should work correctly when model_info is None."""
|
||||
kwargs = {
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
model_info=None,
|
||||
)
|
||||
|
||||
assert entry["litellm_provider"] == "openai"
|
||||
assert entry["input_cost_per_token"] == 0.001
|
||||
assert entry["output_cost_per_token"] == 0.002
|
||||
assert "mode" not in entry
|
||||
|
||||
|
||||
def test_register_model_receives_cache_pricing_fields():
|
||||
"""End-to-end: when register_model is called with a full pricing entry,
|
||||
the cache pricing fields should be present in litellm.model_cost."""
|
||||
|
|
@ -192,23 +73,6 @@ def test_register_model_receives_cache_pricing_fields():
|
|||
litellm.model_cost.pop(model_key, None)
|
||||
|
||||
|
||||
def test_build_custom_pricing_entry_time_based():
|
||||
"""Time-based pricing fields should be included correctly."""
|
||||
kwargs = {
|
||||
"input_cost_per_second": 0.01,
|
||||
"output_cost_per_second": 0.02,
|
||||
}
|
||||
|
||||
entry = _build_custom_pricing_entry(
|
||||
custom_llm_provider="openai",
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
assert entry["litellm_provider"] == "openai"
|
||||
assert entry["input_cost_per_second"] == 0.01
|
||||
assert entry["output_cost_per_second"] == 0.02
|
||||
|
||||
|
||||
def test_register_model_strips_none_litellm_provider():
|
||||
"""``get_model_info`` returns ``litellm_provider: None`` for deployments
|
||||
registered without a provider (e.g. ``Router.add_deployment`` flows).
|
||||
|
|
@ -236,10 +100,7 @@ def test_register_model_strips_none_litellm_provider():
|
|||
registered = litellm.model_cost.get(model_key)
|
||||
assert registered is not None, f"{model_key} should be in model_cost"
|
||||
# The key may be absent entirely, but if present it must not be None.
|
||||
assert (
|
||||
"litellm_provider" not in registered
|
||||
or registered["litellm_provider"] is not None
|
||||
)
|
||||
assert "litellm_provider" not in registered or registered["litellm_provider"] is not None
|
||||
# Downstream consumers must accept this entry for any provider,
|
||||
# mirroring what the cost calculator does.
|
||||
assert _check_provider_match(registered, "openai") is True
|
||||
|
|
@ -297,9 +158,7 @@ def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeyp
|
|||
# The strip must have removed the None-valued provider that
|
||||
# ``get_model_info`` returned. The key may be absent entirely, but
|
||||
# it must never be present with value ``None``.
|
||||
assert "litellm_provider" not in registered or (
|
||||
registered["litellm_provider"] is not None
|
||||
), (
|
||||
assert "litellm_provider" not in registered or (registered["litellm_provider"] is not None), (
|
||||
"register_model failed to strip litellm_provider=None returned "
|
||||
f"by get_model_info, got {registered.get('litellm_provider')!r}"
|
||||
)
|
||||
|
|
@ -360,14 +219,8 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa
|
|||
)
|
||||
|
||||
registered = litellm.model_cost[registered_key]
|
||||
assert (
|
||||
registered.get("cache_creation_input_token_cost")
|
||||
== builtin["cache_creation_input_token_cost"]
|
||||
)
|
||||
assert (
|
||||
registered.get("cache_read_input_token_cost")
|
||||
== builtin["cache_read_input_token_cost"]
|
||||
)
|
||||
assert registered.get("cache_creation_input_token_cost") == builtin["cache_creation_input_token_cost"]
|
||||
assert registered.get("cache_read_input_token_cost") == builtin["cache_read_input_token_cost"]
|
||||
assert registered["litellm_provider"] == "bedrock"
|
||||
|
||||
usage = Usage(
|
||||
|
|
@ -431,8 +284,7 @@ def test_register_model_warns_when_no_builtin_match_for_cache_pricing(caplog):
|
|||
)
|
||||
|
||||
assert any(
|
||||
registered_key in record.message
|
||||
and "cache_creation_input_token_cost" in record.message
|
||||
registered_key in record.message and "cache_creation_input_token_cost" in record.message
|
||||
for record in caplog.records
|
||||
), "expected a warning naming the unmapped key and the cache cost fields"
|
||||
finally:
|
||||
|
|
@ -619,17 +471,14 @@ def test_register_model_router_add_deployment_custom_pricing_applies():
|
|||
# the deployment's provider.
|
||||
from litellm.utils import _check_provider_match
|
||||
|
||||
registered_keys = [
|
||||
k for k in (deployment_model, model_key) if k in litellm.model_cost
|
||||
]
|
||||
registered_keys = [k for k in (deployment_model, model_key) if k in litellm.model_cost]
|
||||
assert registered_keys, (
|
||||
"Router.add_deployment did not register custom pricing for "
|
||||
f"{model_key} / {deployment_model}"
|
||||
f"Router.add_deployment did not register custom pricing for {model_key} / {deployment_model}"
|
||||
)
|
||||
for k in registered_keys:
|
||||
assert (
|
||||
_check_provider_match(litellm.model_cost[k], "openai") is True
|
||||
), f"custom pricing for {k} was dropped by _check_provider_match"
|
||||
assert _check_provider_match(litellm.model_cost[k], "openai") is True, (
|
||||
f"custom pricing for {k} was dropped by _check_provider_match"
|
||||
)
|
||||
finally:
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
litellm.model_cost.pop(deployment_model, None)
|
||||
|
|
@ -646,12 +495,8 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
"""
|
||||
shared_key = "openai/text-embedding-3-small"
|
||||
deployment_id = "lit3991-wildcard-embed-zero"
|
||||
snapshot = _snapshot_model_cost_entries(
|
||||
[shared_key, "text-embedding-3-small", deployment_id]
|
||||
)
|
||||
builtin_input_cost = litellm.get_model_info(model=shared_key)[
|
||||
"input_cost_per_token"
|
||||
]
|
||||
snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id])
|
||||
builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"]
|
||||
assert builtin_input_cost > 0
|
||||
|
||||
try:
|
||||
|
|
@ -666,12 +511,10 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
mock_response=[0.1, 0.2],
|
||||
)
|
||||
|
||||
assert (
|
||||
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
|
||||
== builtin_input_cost
|
||||
), "wildcard deployment's zero pricing leaked into the shared model_cost key"
|
||||
assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0
|
||||
assert litellm.model_cost[deployment_id]["output_cost_per_token"] == 0.0
|
||||
assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, (
|
||||
"wildcard deployment's zero pricing leaked into the shared model_cost key"
|
||||
)
|
||||
assert deployment_id not in litellm.model_cost
|
||||
|
||||
sibling_response = litellm.embedding(
|
||||
model=shared_key,
|
||||
|
|
@ -679,31 +522,33 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
api_key="fake-key",
|
||||
mock_response=[0.1, 0.2],
|
||||
)
|
||||
sibling_cost = litellm.completion_cost(
|
||||
completion_response=sibling_response, call_type="embedding"
|
||||
)
|
||||
sibling_cost = litellm.completion_cost(completion_response=sibling_response, call_type="embedding")
|
||||
assert sibling_cost == pytest.approx(10 * builtin_input_cost)
|
||||
finally:
|
||||
_restore_model_cost_entries(snapshot)
|
||||
|
||||
|
||||
def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
|
||||
"""The request that carries custom pricing must still be costed with that
|
||||
pricing (via its deployment id entry), while the shared backend key keeps
|
||||
the built-in rate for siblings.
|
||||
"""
|
||||
"""A Router registers the deployment's pricing under its id when the deployment is added. The routed request
|
||||
is costed from that entry, and the shared backend key keeps the built-in rate for siblings."""
|
||||
shared_key = "openai/text-embedding-3-small"
|
||||
deployment_id = "lit3991-wildcard-embed-custom"
|
||||
override_input_cost = 5e-05
|
||||
snapshot = _snapshot_model_cost_entries(
|
||||
[shared_key, "text-embedding-3-small", deployment_id]
|
||||
)
|
||||
builtin_input_cost = litellm.get_model_info(model=shared_key)[
|
||||
"input_cost_per_token"
|
||||
]
|
||||
snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id])
|
||||
builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"]
|
||||
assert builtin_input_cost != override_input_cost
|
||||
|
||||
try:
|
||||
litellm.register_model(
|
||||
{
|
||||
deployment_id: {
|
||||
"input_cost_per_token": override_input_cost,
|
||||
"output_cost_per_token": override_input_cost * 2,
|
||||
"litellm_provider": "openai",
|
||||
"mode": "embedding",
|
||||
}
|
||||
}
|
||||
)
|
||||
response = litellm.embedding(
|
||||
model=shared_key,
|
||||
input=["hello"],
|
||||
|
|
@ -715,19 +560,10 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id():
|
|||
mock_response=[0.1, 0.2],
|
||||
)
|
||||
|
||||
request_cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=shared_key,
|
||||
custom_llm_provider="openai",
|
||||
call_type="embedding",
|
||||
custom_pricing=True,
|
||||
router_model_id=deployment_id,
|
||||
)
|
||||
assert request_cost == pytest.approx(10 * override_input_cost)
|
||||
assert (
|
||||
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
|
||||
== builtin_input_cost
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
response.usage.prompt_tokens * override_input_cost
|
||||
)
|
||||
assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost
|
||||
finally:
|
||||
_restore_model_cost_entries(snapshot)
|
||||
|
||||
|
|
@ -736,12 +572,8 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
"""Same isolation as the embedding path, exercised through completion()."""
|
||||
shared_key = "openai/gpt-4o-mini"
|
||||
deployment_id = "lit3991-wildcard-chat-zero"
|
||||
snapshot = _snapshot_model_cost_entries(
|
||||
[shared_key, "gpt-4o-mini", deployment_id]
|
||||
)
|
||||
builtin_input_cost = litellm.get_model_info(model=shared_key)[
|
||||
"input_cost_per_token"
|
||||
]
|
||||
snapshot = _snapshot_model_cost_entries([shared_key, "gpt-4o-mini", deployment_id])
|
||||
builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"]
|
||||
assert builtin_input_cost > 0
|
||||
|
||||
try:
|
||||
|
|
@ -756,24 +588,21 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing():
|
|||
mock_response="hello back",
|
||||
)
|
||||
|
||||
assert (
|
||||
litellm.get_model_info(model=shared_key)["input_cost_per_token"]
|
||||
== builtin_input_cost
|
||||
), "wildcard deployment's zero pricing leaked into the shared model_cost key"
|
||||
assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0
|
||||
assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, (
|
||||
"wildcard deployment's zero pricing leaked into the shared model_cost key"
|
||||
)
|
||||
assert deployment_id not in litellm.model_cost
|
||||
finally:
|
||||
_restore_model_cost_entries(snapshot)
|
||||
|
||||
|
||||
def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key():
|
||||
"""Direct SDK calls (no router deployment id in metadata) keep the legacy
|
||||
behavior: custom pricing is registered under ``{provider}/{model}`` and the
|
||||
request is costed with it.
|
||||
"""
|
||||
model_key = "openai/lit3991-direct-sdk-embed-model"
|
||||
override_input_cost = 3e-05
|
||||
def test_embedding_direct_sdk_custom_pricing_prices_the_call_without_writing_the_cost_map():
|
||||
"""A direct SDK call with its own prices is costed with them and leaves ``litellm.model_cost`` alone, so the
|
||||
next call for the same model name, and every other deployment of it, is priced from the cost map again."""
|
||||
model_key: Final = "openai/lit3991-direct-sdk-embed-model"
|
||||
override_input_cost: Final = 3e-05
|
||||
try:
|
||||
response = litellm.embedding(
|
||||
response: Final = litellm.embedding(
|
||||
model=model_key,
|
||||
input=["hello"],
|
||||
api_key="fake-key",
|
||||
|
|
@ -782,63 +611,18 @@ def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key():
|
|||
mock_response=[0.1, 0.2],
|
||||
)
|
||||
|
||||
assert (
|
||||
litellm.model_cost[model_key]["input_cost_per_token"]
|
||||
== override_input_cost
|
||||
assert model_key not in litellm.model_cost
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
response.usage.prompt_tokens * override_input_cost
|
||||
)
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model=model_key,
|
||||
custom_llm_provider="openai",
|
||||
call_type="embedding",
|
||||
custom_pricing=True,
|
||||
)
|
||||
assert cost == pytest.approx(10 * override_input_cost)
|
||||
|
||||
plain: Final = litellm.embedding(model=model_key, input=["hello"], api_key="fake-key", mock_response=[0.1, 0.2])
|
||||
assert plain._hidden_params["response_cost"] != response._hidden_params["response_cost"]
|
||||
finally:
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ["", "converse/", "invoke/"])
|
||||
def test_register_model_prices_a_geo_alias_from_its_builtin_when_the_lookup_fails(
|
||||
monkeypatch: pytest.MonkeyPatch, route: str
|
||||
) -> None:
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
canonical_key: Final = "au.anthropic.claude-sonnet-5"
|
||||
alias: Final = f"bedrock/{route}{canonical_key}"
|
||||
canonical: Final = litellm.model_cost[canonical_key]
|
||||
|
||||
def lookup_tripped_by_a_concurrent_registration(model: str, custom_llm_provider: str | None = None) -> NoReturn:
|
||||
raise RuntimeError("dictionary changed size during iteration")
|
||||
|
||||
with pytest.MonkeyPatch.context() as lookup_patch:
|
||||
lookup_patch.setattr(litellm.utils, "get_model_info", lookup_tripped_by_a_concurrent_registration)
|
||||
litellm.register_model(
|
||||
{alias: {"litellm_provider": "bedrock", "mode": "chat", "supports_vision": True}},
|
||||
persist_across_reloads=False,
|
||||
)
|
||||
|
||||
usage: Final = Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100)
|
||||
alias_cost: Final = litellm.completion_cost(
|
||||
completion_response=ModelResponse(model=alias, usage=usage), model=alias, custom_llm_provider="bedrock"
|
||||
)
|
||||
canonical_cost: Final = litellm.completion_cost(
|
||||
completion_response=ModelResponse(model=canonical_key, usage=usage),
|
||||
model=canonical_key,
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
try:
|
||||
assert alias_cost == canonical_cost > 0
|
||||
assert litellm.get_model_info(alias)["input_cost_per_token"] == canonical["input_cost_per_token"]
|
||||
assert litellm.get_model_info(alias)["supports_vision"] is True
|
||||
finally:
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_register_model_keeps_a_catalog_rows_own_prices_when_the_lookup_fails(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
geo_key: Final = "au.anthropic.lit-5853-geo-model"
|
||||
geo_row: Final = {
|
||||
|
|
@ -1407,10 +1191,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing():
|
|||
object stays untouched, and the caller's incoming nested dict is never
|
||||
inserted by reference into the merged result.
|
||||
"""
|
||||
from litellm.utils import update_dictionary
|
||||
|
||||
from typing import cast
|
||||
|
||||
from litellm.utils import update_dictionary
|
||||
|
||||
existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"})
|
||||
existing = {"off_peak_pricing": existing_nested}
|
||||
incoming_nested = cast(
|
||||
|
|
@ -1609,12 +1393,12 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing():
|
|||
del router
|
||||
|
||||
|
||||
def test_completion_registers_cost_per_second_pricing():
|
||||
def test_completion_prices_cost_per_second_from_the_call_without_writing_the_cost_map():
|
||||
model_key: Final = "openai/test-cost-per-second-registration"
|
||||
original_entries: Final = _snapshot_model_cost_entries([model_key])
|
||||
|
||||
try:
|
||||
litellm.completion(
|
||||
response: Final = litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
|
|
@ -1622,7 +1406,11 @@ def test_completion_registers_cost_per_second_pricing():
|
|||
mock_response="hello back",
|
||||
)
|
||||
|
||||
assert litellm.model_cost[model_key]["cost_per_second"] == 0.02
|
||||
assert model_key not in litellm.model_cost
|
||||
assert response._hidden_params["response_cost"] > 0
|
||||
assert litellm.completion_cost(completion_response=response, custom_cost_per_second=0.02) == pytest.approx(
|
||||
0.02 * response._response_ms / 1000
|
||||
)
|
||||
finally:
|
||||
_restore_model_cost_entries(original_entries)
|
||||
|
||||
|
|
@ -1676,31 +1464,238 @@ def test_register_model_scopes_builtin_match_to_the_given_provider():
|
|||
_restore_model_cost_entries(model_cost_entries)
|
||||
|
||||
|
||||
def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider():
|
||||
from litellm.main import _register_custom_pricing_for_request
|
||||
def _audio_reply() -> ModelResponse:
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=1000,
|
||||
total_tokens=2000,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=1000, text_tokens=0),
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=1000, text_tokens=0),
|
||||
)
|
||||
return ModelResponse(model="gpt-audio-2025-08-28", choices=[Choices(message=Message(content="hi"))], usage=usage)
|
||||
|
||||
deployment_id: Final = "baseten/zai-org/glm-5.2"
|
||||
builtin_key: Final = "baseten/zai-org/GLM-5.2"
|
||||
shared_key: Final = "openai/zai-org/GLM-5.2"
|
||||
model_cost_entries: Final = _snapshot_model_cost_entries((deployment_id, builtin_key, shared_key))
|
||||
builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key])
|
||||
openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models)
|
||||
|
||||
_AUDIO_RATES: Final = {
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"output_cost_per_token": 1e-05,
|
||||
"input_cost_per_audio_token": 4e-05,
|
||||
"output_cost_per_audio_token": 8e-05,
|
||||
}
|
||||
|
||||
|
||||
def test_router_call_with_audio_rates_is_priced_per_modality_from_its_registered_deployment():
|
||||
"""A Router registers the deployment's full custom pricing when the deployment is added. The per-call flat
|
||||
prices must not override that entry, or 1000 audio input plus 1000 audio output tokens would bill at the text
|
||||
rates ($0.0125) instead of the audio rates ($0.12)."""
|
||||
deployment_id: Final = "audio-rates-deployment"
|
||||
snapshot: Final = _snapshot_model_cost_entries([deployment_id])
|
||||
try:
|
||||
_register_custom_pricing_for_request(
|
||||
model="zai-org/GLM-5.2",
|
||||
custom_llm_provider="openai",
|
||||
kwargs={
|
||||
"input_cost_per_token": 0.00000096,
|
||||
"output_cost_per_token": 0.00000302,
|
||||
"metadata": {"model_info": {"id": deployment_id}},
|
||||
},
|
||||
model_info={"mode": "chat"},
|
||||
litellm.register_model({deployment_id: {**_AUDIO_RATES, "litellm_provider": "openai", "mode": "chat"}})
|
||||
response: Final = litellm.completion(
|
||||
model="openai/gpt-audio-2025-08-28",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
metadata={"model_info": {"id": deployment_id}},
|
||||
mock_response=_audio_reply(),
|
||||
**_AUDIO_RATES,
|
||||
)
|
||||
|
||||
entry: Final = litellm.model_cost[deployment_id]
|
||||
assert entry["input_cost_per_token"] == 0.00000096
|
||||
assert entry["output_cost_per_token"] == 0.00000302
|
||||
assert litellm.model_cost[builtin_key] == builtin_row_before
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(1000 * 4e-05 + 1000 * 8e-05)
|
||||
finally:
|
||||
_restore_model_cost_entries(model_cost_entries)
|
||||
litellm.open_ai_chat_completion_models.intersection_update(openai_models_before)
|
||||
_restore_model_cost_entries(snapshot)
|
||||
|
||||
|
||||
def test_direct_call_with_flat_rates_is_priced_from_them_without_writing_the_cost_map():
|
||||
model_key: Final = "openai/lit-direct-flat-rates-model"
|
||||
try:
|
||||
response: Final = litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
mock_response=_audio_reply(),
|
||||
input_cost_per_token=2.5e-06,
|
||||
output_cost_per_token=1e-05,
|
||||
)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(1000 * 2.5e-06 + 1000 * 1e-05)
|
||||
assert model_key not in litellm.model_cost
|
||||
finally:
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_direct_call_with_audio_rates_is_priced_per_modality_without_writing_the_cost_map():
|
||||
model_key: Final = "openai/lit-direct-audio-rates-model"
|
||||
try:
|
||||
response: Final = litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
mock_response=_audio_reply(),
|
||||
input_cost_per_token=2.5e-06,
|
||||
output_cost_per_token=1e-05,
|
||||
input_cost_per_audio_token=4e-05,
|
||||
output_cost_per_audio_token=8e-05,
|
||||
)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(1000 * 4e-05 + 1000 * 8e-05)
|
||||
assert model_key not in litellm.model_cost
|
||||
finally:
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_direct_call_with_directional_per_second_rates_is_priced_from_them_without_writing_the_cost_map():
|
||||
model_key: Final = "openai/lit-direct-per-second-model"
|
||||
try:
|
||||
response: Final = litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
mock_response="ok",
|
||||
input_cost_per_second=0.004,
|
||||
)
|
||||
assert litellm.completion_cost(completion_response=response, custom_cost_per_second=0.004) == pytest.approx(
|
||||
0.004 * response._response_ms / 1000
|
||||
)
|
||||
assert response._hidden_params["response_cost"] > 0
|
||||
assert model_key not in litellm.model_cost
|
||||
finally:
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_router_call_with_request_prices_bills_them_over_the_deployment_rates():
|
||||
from litellm import Router
|
||||
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "lit-request-override",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.4-mini",
|
||||
"api_key": "fake-key",
|
||||
"mock_response": "ok",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"input_cost_per_audio_token": 4e-05,
|
||||
"output_cost_per_audio_token": 8e-05,
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
overridden: Final = router.completion(
|
||||
model="lit-request-override",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
input_cost_per_token=5e-05,
|
||||
output_cost_per_token=9e-05,
|
||||
)
|
||||
assert overridden._hidden_params["response_cost"] == pytest.approx(
|
||||
overridden.usage.prompt_tokens * 5e-05 + overridden.usage.completion_tokens * 9e-05
|
||||
)
|
||||
plain: Final = router.completion(model="lit-request-override", messages=[{"role": "user", "content": "hello"}])
|
||||
assert plain._hidden_params["response_cost"] == pytest.approx(
|
||||
plain.usage.prompt_tokens * 1e-06 + plain.usage.completion_tokens * 2e-06
|
||||
)
|
||||
|
||||
|
||||
def test_direct_call_flat_rates_override_the_catalogs_tiers_per_direction():
|
||||
model_key: Final = "openai/lit-tiered-catalog-model"
|
||||
litellm.register_model(
|
||||
{
|
||||
model_key: {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"tiered_pricing": [
|
||||
{"range": [0, 128000], "input_cost_per_token": 3e-07, "output_cost_per_token": 6e-07},
|
||||
{"range": [128000, 1000000], "input_cost_per_token": 6e-07, "output_cost_per_token": 1.2e-06},
|
||||
],
|
||||
}
|
||||
},
|
||||
persist_across_reloads=False,
|
||||
)
|
||||
try:
|
||||
response: Final = litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
mock_response="ok",
|
||||
input_cost_per_token=1e-05,
|
||||
output_cost_per_token=2e-05,
|
||||
)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
response.usage.prompt_tokens * 1e-05 + response.usage.completion_tokens * 2e-05
|
||||
)
|
||||
input_only: Final = litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
mock_response="ok",
|
||||
input_cost_per_token=1e-05,
|
||||
)
|
||||
assert input_only._hidden_params["response_cost"] == pytest.approx(
|
||||
input_only.usage.prompt_tokens * 1e-05 + input_only.usage.completion_tokens * 6e-07
|
||||
)
|
||||
assert "input_cost_per_token" not in litellm.model_cost[model_key]
|
||||
finally:
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_direct_call_with_only_a_per_second_rate_on_a_mapped_model_is_billed_per_second():
|
||||
model_key: Final = "openai/lit-mapped-token-rates-model"
|
||||
litellm.register_model(
|
||||
{
|
||||
model_key: {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
}
|
||||
},
|
||||
persist_across_reloads=False,
|
||||
)
|
||||
try:
|
||||
response: Final = litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
mock_response="ok",
|
||||
cost_per_second=0.02,
|
||||
)
|
||||
token_rate_cost: Final = response.usage.prompt_tokens * 1e-06 + response.usage.completion_tokens * 2e-06
|
||||
assert response._hidden_params["response_cost"] != pytest.approx(token_rate_cost)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(0.02 * response._response_ms / 1000, rel=0.25)
|
||||
assert litellm.model_cost[model_key]["input_cost_per_token"] == 1e-06
|
||||
finally:
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
||||
def test_direct_call_flat_rate_applies_to_the_callers_own_tiers_not_the_catalogs():
|
||||
model_key: Final = "openai/lit-caller-tiers-model"
|
||||
litellm.register_model(
|
||||
{
|
||||
model_key: {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"tiered_pricing": [
|
||||
{"range": [0, 1000000], "input_cost_per_token": 3e-07, "output_cost_per_token": 6e-07}
|
||||
],
|
||||
}
|
||||
},
|
||||
persist_across_reloads=False,
|
||||
)
|
||||
try:
|
||||
response: Final = litellm.completion(
|
||||
model=model_key,
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
api_key="fake-key",
|
||||
mock_response="ok",
|
||||
input_cost_per_token=1e-05,
|
||||
tiered_pricing=[{"range": [0, 1000000], "input_cost_per_token": 5e-06, "output_cost_per_token": 9e-06}],
|
||||
)
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(
|
||||
response.usage.prompt_tokens * 1e-05 + response.usage.completion_tokens * 9e-06
|
||||
)
|
||||
assert litellm.model_cost[model_key]["tiered_pricing"][0]["output_cost_per_token"] == 6e-07
|
||||
finally:
|
||||
litellm.model_cost.pop(model_key, None)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
|
|
|||
|
|
@ -2393,7 +2393,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None:
|
|||
|
||||
@respx.mock
|
||||
@pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map")
|
||||
def test_completion_registers_test_owned_model_prices() -> None:
|
||||
def test_completion_custom_prices_price_the_call_without_writing_the_cost_map() -> None:
|
||||
model: Final = "migration-owned-pricing-model"
|
||||
messages: Final = [{"role": "user", "content": "price control"}]
|
||||
litellm.register_model(
|
||||
|
|
@ -2436,8 +2436,9 @@ def test_completion_registers_test_owned_model_prices() -> None:
|
|||
)
|
||||
|
||||
assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages}
|
||||
assert litellm.model_cost[model]["input_cost_per_token"] == 0.001
|
||||
assert litellm.model_cost[model]["output_cost_per_token"] == 0.002
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(8 * 0.001 + 5 * 0.002)
|
||||
assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001
|
||||
assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002
|
||||
assert response._hidden_params["response_cost"] == pytest.approx(0.018)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue