From 65950dba8b2dcfb5a52a1977a221885f0ec20aa8 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 3 Nov 2026 16:03:34 -0800 Subject: [PATCH] fix(main)!: carry the full per-call price entry on the logging object instead of writing litellm.model_cost (#49532) * fix(main): price per-request custom pricing from the logging object instead of writing litellm.model_cost * fix(main)!: carry the full per-call price entry on the logging object instead of writing litellm.model_cost Direct SDK calls build one ModelInfo from the cost map row plus every custom pricing field on the call and price from it, so audio, tiered and directional per-second rates apply without touching litellm.model_cost. Router calls keep their registered deployment entry. completion_cost(completion_response=...) alone no longer sees per-call prices, pass custom_cost_per_second or custom_cost_per_token * test(cost): assert the exact per-second charge for per-call cost_per_second * fix(main): price Router request overrides and flat rates over inherited tiers in the per-call entry A Router call that sets its own price is priced from it over the registered deployment entry, PTU zeroing applied as at registration. Flat token rates on a call drop an inherited tiered_pricing table, which generic_cost_per_token would otherwise prefer * fix(main): keep tier rates for directions a call does not override and bill per-second-only calls per second A flat rate set on a call is written into every tier for that direction instead of dropping the tier table, so the other direction keeps its tier price. A call that sets only a per-second rate on a model with catalog token rates clears those token rates for the call so the per-second path applies * fix(main): apply a call's flat rate to its own tiered_pricing table when it supplies one * refactor(main): move per-call price entry resolution into litellm_core_utils/per_call_pricing The resolution logic is pure cost-map work with no dependency on dispatch, so it lives next to ptu_pricing and main.py keeps only the call sites in completion() and embedding() Co-Authored-By: Claude Opus 5.5 * refactor(cost): rename per_call_model_info parameter to requested_model_info_override Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- litellm/cost_calculator.py | 30 +- litellm/litellm_core_utils/litellm_logging.py | 6 + .../litellm_core_utils/per_call_pricing.py | 104 +++ litellm/main.py | 127 +--- .../test_sagemaker_completion_handler.py | 2 +- .../test_register_model_custom_pricing.py | 605 +++++++++--------- tests/unit/test_utils.py | 7 +- 7 files changed, 448 insertions(+), 433 deletions(-) create mode 100644 litellm/litellm_core_utils/per_call_pricing.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7bd162b4f45..27a954f5f85 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -442,6 +442,14 @@ def _per_second_pricing_cost( model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model return None + return _per_second_pricing_cost_from_info(model_info, response_time_ms, audio_seconds) + + +def _per_second_pricing_cost_from_info( + model_info: ModelInfoBase, + response_time_ms: float | None, + audio_seconds: float = 0.0, +) -> tuple[float, float] | None: if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): return None cost_per_second: Final = model_info.get("cost_per_second") @@ -464,7 +472,7 @@ def _per_second_pricing_cost( ) verbose_logger.debug( "For model=%s - cost_per_second: %s; response time: %s", - model, + model_info.get("key"), resolved_cost_per_second, response_time_ms, ) @@ -487,6 +495,7 @@ def cost_per_token( ### CUSTOM PRICING ### custom_cost_per_token: CostPerToken | None = None, custom_cost_per_second: float | None = None, + requested_model_info_override: ModelInfo | None = None, ### NUMBER OF QUERIES ### number_of_queries: int | None = None, ### USAGE OBJECT ### @@ -598,6 +607,20 @@ def cost_per_token( if response_cost is not None: return response_cost[0], response_cost[1] + if requested_model_info_override is not None: + per_second: Final = _per_second_pricing_cost_from_info( + requested_model_info_override, response_time_ms, audio_transcription_file_duration + ) + if per_second is not None: + return per_second + return generic_cost_per_token( + model=model, + usage=usage_block, + custom_llm_provider=custom_llm_provider or "", + service_tier=service_tier, + model_info=requested_model_info_override, + ) + model_cost_ref: Final = litellm.model_cost # Only callers that explicitly pass `custom_llm_provider` get the # dedup/prefix-join treatment. When provider is omitted, preserve legacy @@ -1747,6 +1770,7 @@ def completion_cost( ### CUSTOM PRICING ### custom_cost_per_token: CostPerToken | None = None, custom_cost_per_second: float | None = None, + requested_model_info_override: ModelInfo | None = None, optional_params: dict | None = None, custom_pricing: bool | None = None, base_model: str | None = None, @@ -1800,6 +1824,7 @@ def completion_cost( quality=quality, n=n, custom_cost_per_token=custom_cost_per_token, + requested_model_info_override=requested_model_info_override, custom_cost_per_second=custom_cost_per_second, optional_params=optional_params, custom_pricing=custom_pricing, @@ -2274,6 +2299,7 @@ def completion_cost( region_name=None if explicit_pricing else region_name, custom_cost_per_second=custom_cost_per_second, custom_cost_per_token=custom_cost_per_token, + requested_model_info_override=requested_model_info_override, prompt_characters=prompt_characters, completion_characters=completion_characters, cache_creation_input_tokens=cache_creation_input_tokens, @@ -2508,6 +2534,7 @@ def response_cost_calculator( region_name: str | None = None, custom_cost_per_token: CostPerToken | None = None, custom_cost_per_second: float | None = None, + requested_model_info_override: ModelInfo | None = None, ) -> float: """ Returns @@ -2564,6 +2591,7 @@ def response_cost_calculator( custom_pricing=custom_pricing, base_model=base_model, custom_cost_per_token=custom_cost_per_token, + requested_model_info_override=requested_model_info_override, custom_cost_per_second=custom_cost_per_second, prompt=prompt, standard_built_in_tools_params=standard_built_in_tools_params, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index dce98185cab..6e79f8e56ef 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -722,6 +722,7 @@ class Logging(LiteLLMLoggingBaseClass): custom_pricing: bool = False custom_cost_per_token: CostPerToken | None = None custom_cost_per_second: float | None = None + custom_model_info: ModelInfo | None = None replaced_global_callback_keys: frozenset[str] = frozenset() stream_options = None litellm_request_debug: bool = False @@ -2227,6 +2228,7 @@ class Logging(LiteLLMLoggingBaseClass): "custom_pricing": custom_pricing, "custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None, "custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None, + "requested_model_info_override": self.custom_model_info if custom_pricing else None, "prompt": prompt, "standard_built_in_tools_params": self.standard_built_in_tools_params, "router_model_id": router_model_id, @@ -2423,6 +2425,10 @@ class Logging(LiteLLMLoggingBaseClass): additional_headers=additional_headers, ) + def set_per_call_pricing(self, custom_model_info: ModelInfo | None) -> None: + """Price this call from its own full price entry instead of ``litellm.model_cost``.""" + self.custom_model_info = custom_model_info + def custom_pricing_for(self, result: object) -> bool: if self.spilled_over_ptu_for(result): return False diff --git a/litellm/litellm_core_utils/per_call_pricing.py b/litellm/litellm_core_utils/per_call_pricing.py new file mode 100644 index 00000000000..686639dc10f --- /dev/null +++ b/litellm/litellm_core_utils/per_call_pricing.py @@ -0,0 +1,104 @@ +"""The price entry for a single call that sets its own custom pricing, built without writing ``litellm.model_cost``.""" + +from collections.abc import Mapping +from typing import Final, Literal + +from pydantic import TypeAdapter + +import litellm +from litellm.litellm_core_utils.ptu_pricing import zeroed_ptu_pricing +from litellm.types.utils import CustomPricingLiteLLMParams, ModelInfo + +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_MODEL_INFO: Final = TypeAdapter(ModelInfo) + + +def _router_deployment_model_info(kwargs: Mapping[str, object]) -> Mapping[str, object] | None: + """A Router stamps the deployment's ``model_info`` (with its ``id``) into the call's metadata and registers that + deployment's full custom pricing under that id when the deployment is added.""" + for metadata_key in ("litellm_metadata", "metadata"): + metadata = kwargs.get(metadata_key) + if not isinstance(metadata, Mapping): + continue + model_info = _STR_MAPPING.validate_python(metadata).get("model_info") + if isinstance(model_info, Mapping) and isinstance(model_info.get("id"), str): + return _STR_MAPPING.validate_python(model_info) + return None + + +def per_call_model_info( + model: str, custom_llm_provider: str, kwargs: Mapping[str, object], mode: Literal["chat", "embedding"] +) -> ModelInfo | None: + """The full price entry for one call, or None when the cost map (or the Router's registered deployment entry) + already prices it. The cost map row for the model is overlaid with the registered deployment entry and then with + the custom pricing fields the request itself set, so a Router call pays request overrides over deployment prices. + The entry travels on the logging object and is never written into ``litellm.model_cost``, so the next call for the + same model name is priced from the cost map again.""" + deployment: Final = _router_deployment_model_info(kwargs) + registered: Final = litellm.model_cost.get(str(deployment["id"])) if deployment is not None else None + custom: Final = { + field: kwargs[field] + for field in CustomPricingLiteLLMParams.model_fields + if kwargs.get(field) is not None and (registered is None or registered.get(field) != kwargs[field]) + } + if not custom: + return None + unmapped: Final[Mapping[str, object]] = { + "key": model, + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": None, + "output_cost_per_token": None, + "litellm_provider": custom_llm_provider, + "mode": mode, + "supported_openai_params": None, + } + row: Final = _catalog_row(model, custom_llm_provider) or unmapped + catalog: Final = ( + {**row, **{key: value for key, value in registered.items() if value is not None}} if registered else row + ) + zeroed: Final = zeroed_ptu_pricing(deployment, custom) if deployment is not None else None + return _MODEL_INFO.validate_python( + { + **catalog, + **_token_rates_cleared_for_per_second_overrides(custom), + **custom, + **_tiers_with_flat_overrides(catalog, custom), + **(zeroed or {}), + "mode": catalog.get("mode") or mode, + } + ) + + +_FLAT_TOKEN_RATE_FIELDS: Final = ("input_cost_per_token", "output_cost_per_token") +_PER_SECOND_RATE_FIELDS: Final = ("cost_per_second", "input_cost_per_second", "output_cost_per_second") +_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +def _token_rates_cleared_for_per_second_overrides(custom: Mapping[str, object]) -> Mapping[str, object]: + """Token rates in the entry make the cost calculator skip per-second pricing, so a caller who sets only a + per-second rate gets the catalog's token rates and tier table cleared for this call.""" + if not any(key in custom for key in _PER_SECOND_RATE_FIELDS) or any( + key in custom for key in _FLAT_TOKEN_RATE_FIELDS + ): + return {} + return {"input_cost_per_token": None, "output_cost_per_token": None, "tiered_pricing": None} + + +def _tiers_with_flat_overrides(catalog: Mapping[str, object], custom: Mapping[str, object]) -> Mapping[str, object]: + """A ``tiered_pricing`` table wins over flat rates in the cost calculator, so a flat rate the caller set is written + into every tier of the table in force (the caller's own, else the inherited one) for that direction, and the other + direction keeps each tier's own rate.""" + overrides: Final = {key: custom[key] for key in _FLAT_TOKEN_RATE_FIELDS if key in custom} + tiers: Final = custom.get("tiered_pricing", catalog.get("tiered_pricing")) + if not overrides or not isinstance(tiers, (list, tuple)) or not tiers: + return {} + return {"tiered_pricing": [{**tier, **overrides} for tier in _TIERS.validate_python(tiers)]} + + +def _catalog_row(model: str, custom_llm_provider: str) -> ModelInfo | None: + try: + return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model + return None diff --git a/litellm/main.py b/litellm/main.py index 45177f4223e..86da08a23ab 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -111,6 +111,7 @@ from litellm.litellm_core_utils.mock_functions import ( mock_embedding, mock_image_generation, ) +from litellm.litellm_core_utils.per_call_pricing import per_call_model_info from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response as get_content_from_model_response, # noqa: PLC0414 # public re-export ) @@ -167,7 +168,6 @@ from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_s from litellm.types.utils import ( AgenticLoopParams, BaseUrlRoutingHint, - CustomPricingLiteLLMParams, ModelResponseStream, ProviderRoutingHints, RawRequestTypedDict, @@ -1323,7 +1323,6 @@ from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the o ) from litellm.responses.bridge_check import ( bridge_reasoning_effort, - keep_registered_mode, str_keyed_mapping, ) from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site @@ -1429,46 +1428,6 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]: return values -def _build_custom_pricing_entry( - custom_llm_provider: str, - kwargs: dict, - model_info: dict | None = None, -) -> dict: - """Build a complete model cost entry from kwargs and model_info. - - Collects all CustomPricingLiteLLMParams fields present in kwargs and - merges metadata from model_info (mode, supports_prompt_caching, max_tokens) - so that register_model() receives the full pricing configuration. - """ - entry: Final[dict] = {"litellm_provider": custom_llm_provider} - - for field_name in CustomPricingLiteLLMParams.model_fields: - value = kwargs.get(field_name) - if value is not None: - entry[field_name] = value - - if model_info and isinstance(model_info, dict): - for key in ("mode", "supports_prompt_caching", "max_tokens"): - if key in model_info and model_info[key] is not None: - entry.setdefault(key, model_info[key]) - - return entry - - -def _get_router_deployment_id(kwargs: dict) -> str | None: - for metadata_key in ("litellm_metadata", "metadata"): - metadata = kwargs.get(metadata_key) or {} - if not isinstance(metadata, dict): - continue - deployment_model_info = metadata.get("model_info") or {} - if not isinstance(deployment_model_info, dict): - continue - deployment_id = deployment_model_info.get("id") - if deployment_id is not None: - return str(deployment_id) - return None - - def _record_agentic_loop_params( logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool ) -> None: @@ -1479,53 +1438,6 @@ def _record_agentic_loop_params( ) -def _register_custom_pricing_for_request( - model: str, - custom_llm_provider: str, - kwargs: dict, - model_info: dict | None, -) -> None: - """Register per-request custom pricing in litellm.model_cost. - - Router-originated requests (identified by the deployment id the router puts - in metadata) get their full pricing registered under that unique id only; - the shared ``{provider}/{model}`` key receives the entry with pricing fields - stripped, mirroring Router._create_deployment. This keeps one deployment's - pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in - pricing used by sibling deployments of the same backend model. Direct SDK - calls keep the legacy behavior of registering the shared key with pricing. - Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on - deployment registration, since a call routes by its own ``model_info``. - """ - entry: Final = _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - shared_key: Final = f"{custom_llm_provider}/{model}" - deployment_id: Final = _get_router_deployment_id(kwargs) - if deployment_id is None: - litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False) - return - shared_entry: Final = keep_registered_mode( - shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry) - ) - litellm.register_model( - {deployment_id: entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - litellm.register_model( - {shared_key: shared_entry}, - persist_across_reloads=False, - warning_display_name=shared_key, - custom_llm_provider=custom_llm_provider, - add_to_provider_model_sets=False, - ) - - def _dispatch_metadata(ctx: CompletionDispatchContext) -> Mapping[str, object] | None: return ctx.metadata @@ -5787,9 +5699,9 @@ def completion( ### CUSTOM MODEL COST ### input_cost_per_token: Final = kwargs.get("input_cost_per_token", None) output_cost_per_token: Final = kwargs.get("output_cost_per_token", None) - cost_per_second: Final = kwargs.get("cost_per_second", None) input_cost_per_second: Final = kwargs.get("input_cost_per_second", None) output_cost_per_second: Final = kwargs.get("output_cost_per_second", None) + cost_per_second: Final = kwargs.get("cost_per_second", None) ### CUSTOM PROMPT TEMPLATE ### initial_prompt_value: Final = kwargs.get("initial_prompt_value", None) roles: Final = kwargs.get("roles", None) @@ -5987,19 +5899,7 @@ def completion( "aws_region_name", None ) # support region-based pricing for bedrock - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) + logging.set_per_call_pricing(per_call_model_info(model, custom_llm_provider, kwargs, "chat")) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token: @@ -7006,12 +6906,6 @@ def embedding( headers.update(proxy_headers) except Exception as e: verbose_logger.warning("Failed to get proxy auth headers: %s", e) - ### CUSTOM MODEL COST ### - input_cost_per_token: Final = kwargs.get("input_cost_per_token", None) - output_cost_per_token: Final = kwargs.get("output_cost_per_token", None) - cost_per_second: Final = kwargs.get("cost_per_second", None) - input_cost_per_second: Final = kwargs.get("input_cost_per_second", None) - output_cost_per_second: Final = kwargs.get("output_cost_per_second", None) openai_params: Final = [ "user", "dimensions", @@ -7058,21 +6952,8 @@ def embedding( **non_default_params, ) - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - (input_cost_per_token is not None and output_cost_per_token is not None) - or input_cost_per_second is not None - or output_cost_per_second is not None - or cost_per_second is not None - ): - _register_custom_pricing_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=kwargs.get("model_info"), - ) - logging: Final[LiteLLMLoggingObj] = litellm_logging_obj + logging.set_per_call_pricing(per_call_model_info(model, custom_llm_provider, kwargs, "embedding")) logging.update_environment_variables( model=model, user=user, diff --git a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py index 5e40c2ebb60..4c253a10116 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py @@ -549,7 +549,7 @@ async def test_completion_sagemaker( assert route.called assert response.choices[0].message.content == "SageMaker reply" - assert litellm.completion_cost(completion_response=response) == pytest.approx(rate) + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate) @pytest.mark.asyncio diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 35fbd28eae7..afff995aa69 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -17,10 +17,16 @@ from typing import Final, NoReturn import pytest - import litellm from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map -from litellm.main import _build_custom_pricing_entry +from litellm.types.utils import ( + Choices, + CompletionTokensDetailsWrapper, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, +) from litellm.utils import _invalidate_model_cost_lowercase_map @@ -37,131 +43,6 @@ def _restore_model_cost_entries(original_entries): _invalidate_model_cost_lowercase_map() -def test_build_custom_pricing_entry_includes_all_kwargs_fields(): - """All CustomPricingLiteLLMParams fields present in kwargs should be - included in the resulting entry dict.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - "cache_read_input_token_cost": 0.00025, - "cache_creation_input_token_cost": 0.005, - "output_cost_per_reasoning_token": 0.01, - "input_cost_per_audio_token": 0.003, - "unrelated_kwarg": "should_be_ignored", - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_token"] == 0.001 - assert entry["output_cost_per_token"] == 0.002 - assert entry["cache_read_input_token_cost"] == 0.00025 - assert entry["cache_creation_input_token_cost"] == 0.005 - assert entry["output_cost_per_reasoning_token"] == 0.01 - assert entry["input_cost_per_audio_token"] == 0.003 - assert "unrelated_kwarg" not in entry - - -def test_build_custom_pricing_entry_merges_model_info_metadata(): - """Fields from model_info (mode, supports_prompt_caching, max_tokens) - should be merged into the entry when present.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - model_info = { - "id": "deployment-123", - "mode": "chat", - "supports_prompt_caching": True, - "max_tokens": 128000, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=model_info, - ) - - assert entry["mode"] == "chat" - assert entry["supports_prompt_caching"] is True - assert entry["max_tokens"] == 128000 - - -def test_build_custom_pricing_entry_setdefault_does_not_override_existing(): - """model_info uses setdefault, so it should not override a key that is - already present in the entry dict. Currently CustomPricingLiteLLMParams - and the model_info keys (mode, supports_prompt_caching, max_tokens) do - not overlap, but if they ever do, setdefault ensures the kwargs-sourced - value wins.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - model_info = { - "mode": "chat", - "supports_prompt_caching": True, - "max_tokens": 128000, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=model_info, - ) - - assert entry["mode"] == "chat" - assert entry["supports_prompt_caching"] is True - assert entry["max_tokens"] == 128000 - - # Verify setdefault behavior: if a model_info key already exists in - # the entry (e.g. from a future CustomPricingLiteLLMParams addition), - # setdefault must not overwrite it. - entry["mode"] = "embedding" # simulate pre-existing value - # Re-apply setdefault the same way _build_custom_pricing_entry does - entry.setdefault("mode", model_info["mode"]) - assert entry["mode"] == "embedding" # must NOT revert to "chat" - - -def test_build_custom_pricing_entry_skips_none_values(): - """Fields with None values in kwargs should not be included.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": None, # explicitly None - "cache_read_input_token_cost": None, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["input_cost_per_token"] == 0.001 - assert "output_cost_per_token" not in entry - assert "cache_read_input_token_cost" not in entry - - -def test_build_custom_pricing_entry_handles_no_model_info(): - """Should work correctly when model_info is None.""" - kwargs = { - "input_cost_per_token": 0.001, - "output_cost_per_token": 0.002, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - model_info=None, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_token"] == 0.001 - assert entry["output_cost_per_token"] == 0.002 - assert "mode" not in entry - - def test_register_model_receives_cache_pricing_fields(): """End-to-end: when register_model is called with a full pricing entry, the cache pricing fields should be present in litellm.model_cost.""" @@ -192,23 +73,6 @@ def test_register_model_receives_cache_pricing_fields(): litellm.model_cost.pop(model_key, None) -def test_build_custom_pricing_entry_time_based(): - """Time-based pricing fields should be included correctly.""" - kwargs = { - "input_cost_per_second": 0.01, - "output_cost_per_second": 0.02, - } - - entry = _build_custom_pricing_entry( - custom_llm_provider="openai", - kwargs=kwargs, - ) - - assert entry["litellm_provider"] == "openai" - assert entry["input_cost_per_second"] == 0.01 - assert entry["output_cost_per_second"] == 0.02 - - def test_register_model_strips_none_litellm_provider(): """``get_model_info`` returns ``litellm_provider: None`` for deployments registered without a provider (e.g. ``Router.add_deployment`` flows). @@ -236,10 +100,7 @@ def test_register_model_strips_none_litellm_provider(): registered = litellm.model_cost.get(model_key) assert registered is not None, f"{model_key} should be in model_cost" # The key may be absent entirely, but if present it must not be None. - assert ( - "litellm_provider" not in registered - or registered["litellm_provider"] is not None - ) + assert "litellm_provider" not in registered or registered["litellm_provider"] is not None # Downstream consumers must accept this entry for any provider, # mirroring what the cost calculator does. assert _check_provider_match(registered, "openai") is True @@ -297,9 +158,7 @@ def test_register_model_strips_none_litellm_provider_from_get_model_info(monkeyp # The strip must have removed the None-valued provider that # ``get_model_info`` returned. The key may be absent entirely, but # it must never be present with value ``None``. - assert "litellm_provider" not in registered or ( - registered["litellm_provider"] is not None - ), ( + assert "litellm_provider" not in registered or (registered["litellm_provider"] is not None), ( "register_model failed to strip litellm_provider=None returned " f"by get_model_info, got {registered.get('litellm_provider')!r}" ) @@ -360,14 +219,8 @@ def test_register_model_inherits_builtin_cache_pricing_for_unmapped_key(monkeypa ) registered = litellm.model_cost[registered_key] - assert ( - registered.get("cache_creation_input_token_cost") - == builtin["cache_creation_input_token_cost"] - ) - assert ( - registered.get("cache_read_input_token_cost") - == builtin["cache_read_input_token_cost"] - ) + assert registered.get("cache_creation_input_token_cost") == builtin["cache_creation_input_token_cost"] + assert registered.get("cache_read_input_token_cost") == builtin["cache_read_input_token_cost"] assert registered["litellm_provider"] == "bedrock" usage = Usage( @@ -431,8 +284,7 @@ def test_register_model_warns_when_no_builtin_match_for_cache_pricing(caplog): ) assert any( - registered_key in record.message - and "cache_creation_input_token_cost" in record.message + registered_key in record.message and "cache_creation_input_token_cost" in record.message for record in caplog.records ), "expected a warning naming the unmapped key and the cache cost fields" finally: @@ -619,17 +471,14 @@ def test_register_model_router_add_deployment_custom_pricing_applies(): # the deployment's provider. from litellm.utils import _check_provider_match - registered_keys = [ - k for k in (deployment_model, model_key) if k in litellm.model_cost - ] + registered_keys = [k for k in (deployment_model, model_key) if k in litellm.model_cost] assert registered_keys, ( - "Router.add_deployment did not register custom pricing for " - f"{model_key} / {deployment_model}" + f"Router.add_deployment did not register custom pricing for {model_key} / {deployment_model}" ) for k in registered_keys: - assert ( - _check_provider_match(litellm.model_cost[k], "openai") is True - ), f"custom pricing for {k} was dropped by _check_provider_match" + assert _check_provider_match(litellm.model_cost[k], "openai") is True, ( + f"custom pricing for {k} was dropped by _check_provider_match" + ) finally: litellm.model_cost.pop(model_key, None) litellm.model_cost.pop(deployment_model, None) @@ -646,12 +495,8 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): """ shared_key = "openai/text-embedding-3-small" deployment_id = "lit3991-wildcard-embed-zero" - snapshot = _snapshot_model_cost_entries( - [shared_key, "text-embedding-3-small", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost > 0 try: @@ -666,12 +511,10 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): mock_response=[0.1, 0.2], ) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ), "wildcard deployment's zero pricing leaked into the shared model_cost key" - assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0 - assert litellm.model_cost[deployment_id]["output_cost_per_token"] == 0.0 + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, ( + "wildcard deployment's zero pricing leaked into the shared model_cost key" + ) + assert deployment_id not in litellm.model_cost sibling_response = litellm.embedding( model=shared_key, @@ -679,31 +522,33 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): api_key="fake-key", mock_response=[0.1, 0.2], ) - sibling_cost = litellm.completion_cost( - completion_response=sibling_response, call_type="embedding" - ) + sibling_cost = litellm.completion_cost(completion_response=sibling_response, call_type="embedding") assert sibling_cost == pytest.approx(10 * builtin_input_cost) finally: _restore_model_cost_entries(snapshot) def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): - """The request that carries custom pricing must still be costed with that - pricing (via its deployment id entry), while the shared backend key keeps - the built-in rate for siblings. - """ + """A Router registers the deployment's pricing under its id when the deployment is added. The routed request + is costed from that entry, and the shared backend key keeps the built-in rate for siblings.""" shared_key = "openai/text-embedding-3-small" deployment_id = "lit3991-wildcard-embed-custom" override_input_cost = 5e-05 - snapshot = _snapshot_model_cost_entries( - [shared_key, "text-embedding-3-small", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "text-embedding-3-small", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost != override_input_cost try: + litellm.register_model( + { + deployment_id: { + "input_cost_per_token": override_input_cost, + "output_cost_per_token": override_input_cost * 2, + "litellm_provider": "openai", + "mode": "embedding", + } + } + ) response = litellm.embedding( model=shared_key, input=["hello"], @@ -715,19 +560,10 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): mock_response=[0.1, 0.2], ) - request_cost = litellm.completion_cost( - completion_response=response, - model=shared_key, - custom_llm_provider="openai", - call_type="embedding", - custom_pricing=True, - router_model_id=deployment_id, - ) - assert request_cost == pytest.approx(10 * override_input_cost) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * override_input_cost ) + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost finally: _restore_model_cost_entries(snapshot) @@ -736,12 +572,8 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): """Same isolation as the embedding path, exercised through completion().""" shared_key = "openai/gpt-4o-mini" deployment_id = "lit3991-wildcard-chat-zero" - snapshot = _snapshot_model_cost_entries( - [shared_key, "gpt-4o-mini", deployment_id] - ) - builtin_input_cost = litellm.get_model_info(model=shared_key)[ - "input_cost_per_token" - ] + snapshot = _snapshot_model_cost_entries([shared_key, "gpt-4o-mini", deployment_id]) + builtin_input_cost = litellm.get_model_info(model=shared_key)["input_cost_per_token"] assert builtin_input_cost > 0 try: @@ -756,24 +588,21 @@ def test_completion_router_zero_pricing_does_not_clobber_builtin_pricing(): mock_response="hello back", ) - assert ( - litellm.get_model_info(model=shared_key)["input_cost_per_token"] - == builtin_input_cost - ), "wildcard deployment's zero pricing leaked into the shared model_cost key" - assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.0 + assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost, ( + "wildcard deployment's zero pricing leaked into the shared model_cost key" + ) + assert deployment_id not in litellm.model_cost finally: _restore_model_cost_entries(snapshot) -def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): - """Direct SDK calls (no router deployment id in metadata) keep the legacy - behavior: custom pricing is registered under ``{provider}/{model}`` and the - request is costed with it. - """ - model_key = "openai/lit3991-direct-sdk-embed-model" - override_input_cost = 3e-05 +def test_embedding_direct_sdk_custom_pricing_prices_the_call_without_writing_the_cost_map(): + """A direct SDK call with its own prices is costed with them and leaves ``litellm.model_cost`` alone, so the + next call for the same model name, and every other deployment of it, is priced from the cost map again.""" + model_key: Final = "openai/lit3991-direct-sdk-embed-model" + override_input_cost: Final = 3e-05 try: - response = litellm.embedding( + response: Final = litellm.embedding( model=model_key, input=["hello"], api_key="fake-key", @@ -782,63 +611,18 @@ def test_embedding_direct_sdk_custom_pricing_still_registers_shared_key(): mock_response=[0.1, 0.2], ) - assert ( - litellm.model_cost[model_key]["input_cost_per_token"] - == override_input_cost + assert model_key not in litellm.model_cost + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * override_input_cost ) - cost = litellm.completion_cost( - completion_response=response, - model=model_key, - custom_llm_provider="openai", - call_type="embedding", - custom_pricing=True, - ) - assert cost == pytest.approx(10 * override_input_cost) + + plain: Final = litellm.embedding(model=model_key, input=["hello"], api_key="fake-key", mock_response=[0.1, 0.2]) + assert plain._hidden_params["response_cost"] != response._hidden_params["response_cost"] finally: litellm.model_cost.pop(model_key, None) _invalidate_model_cost_lowercase_map() -@pytest.mark.parametrize("route", ["", "converse/", "invoke/"]) -def test_register_model_prices_a_geo_alias_from_its_builtin_when_the_lookup_fails( - monkeypatch: pytest.MonkeyPatch, route: str -) -> None: - from litellm.types.utils import ModelResponse, Usage - - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) - _invalidate_model_cost_lowercase_map() - canonical_key: Final = "au.anthropic.claude-sonnet-5" - alias: Final = f"bedrock/{route}{canonical_key}" - canonical: Final = litellm.model_cost[canonical_key] - - def lookup_tripped_by_a_concurrent_registration(model: str, custom_llm_provider: str | None = None) -> NoReturn: - raise RuntimeError("dictionary changed size during iteration") - - with pytest.MonkeyPatch.context() as lookup_patch: - lookup_patch.setattr(litellm.utils, "get_model_info", lookup_tripped_by_a_concurrent_registration) - litellm.register_model( - {alias: {"litellm_provider": "bedrock", "mode": "chat", "supports_vision": True}}, - persist_across_reloads=False, - ) - - usage: Final = Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100) - alias_cost: Final = litellm.completion_cost( - completion_response=ModelResponse(model=alias, usage=usage), model=alias, custom_llm_provider="bedrock" - ) - canonical_cost: Final = litellm.completion_cost( - completion_response=ModelResponse(model=canonical_key, usage=usage), - model=canonical_key, - custom_llm_provider="bedrock", - ) - try: - assert alias_cost == canonical_cost > 0 - assert litellm.get_model_info(alias)["input_cost_per_token"] == canonical["input_cost_per_token"] - assert litellm.get_model_info(alias)["supports_vision"] is True - finally: - _invalidate_model_cost_lowercase_map() - - def test_register_model_keeps_a_catalog_rows_own_prices_when_the_lookup_fails(monkeypatch: pytest.MonkeyPatch) -> None: geo_key: Final = "au.anthropic.lit-5853-geo-model" geo_row: Final = { @@ -1407,10 +1191,10 @@ def test_update_dictionary_merges_nested_dicts_without_aliasing(): object stays untouched, and the caller's incoming nested dict is never inserted by reference into the merged result. """ - from litellm.utils import update_dictionary - from typing import cast + from litellm.utils import update_dictionary + existing_nested = cast(dict[str, object], {"hours_utc": "01:00-02:00", 1: "existing"}) existing = {"off_peak_pricing": existing_nested} incoming_nested = cast( @@ -1609,12 +1393,12 @@ def test_completion_cost_applies_off_peak_only_deployment_pricing(): del router -def test_completion_registers_cost_per_second_pricing(): +def test_completion_prices_cost_per_second_from_the_call_without_writing_the_cost_map(): model_key: Final = "openai/test-cost-per-second-registration" original_entries: Final = _snapshot_model_cost_entries([model_key]) try: - litellm.completion( + response: Final = litellm.completion( model=model_key, messages=[{"role": "user", "content": "hello"}], api_key="fake-key", @@ -1622,7 +1406,11 @@ def test_completion_registers_cost_per_second_pricing(): mock_response="hello back", ) - assert litellm.model_cost[model_key]["cost_per_second"] == 0.02 + assert model_key not in litellm.model_cost + assert response._hidden_params["response_cost"] > 0 + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=0.02) == pytest.approx( + 0.02 * response._response_ms / 1000 + ) finally: _restore_model_cost_entries(original_entries) @@ -1676,31 +1464,238 @@ def test_register_model_scopes_builtin_match_to_the_given_provider(): _restore_model_cost_entries(model_cost_entries) -def test_per_request_custom_pricing_scopes_a_colliding_deployment_id_to_its_provider(): - from litellm.main import _register_custom_pricing_for_request +def _audio_reply() -> ModelResponse: + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=1000, + total_tokens=2000, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=1000, text_tokens=0), + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=1000, text_tokens=0), + ) + return ModelResponse(model="gpt-audio-2025-08-28", choices=[Choices(message=Message(content="hi"))], usage=usage) - deployment_id: Final = "baseten/zai-org/glm-5.2" - builtin_key: Final = "baseten/zai-org/GLM-5.2" - shared_key: Final = "openai/zai-org/GLM-5.2" - model_cost_entries: Final = _snapshot_model_cost_entries((deployment_id, builtin_key, shared_key)) - builtin_row_before: Final = copy.deepcopy(litellm.model_cost[builtin_key]) - openai_models_before: Final = frozenset(litellm.open_ai_chat_completion_models) + +_AUDIO_RATES: Final = { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + "input_cost_per_audio_token": 4e-05, + "output_cost_per_audio_token": 8e-05, +} + + +def test_router_call_with_audio_rates_is_priced_per_modality_from_its_registered_deployment(): + """A Router registers the deployment's full custom pricing when the deployment is added. The per-call flat + prices must not override that entry, or 1000 audio input plus 1000 audio output tokens would bill at the text + rates ($0.0125) instead of the audio rates ($0.12).""" + deployment_id: Final = "audio-rates-deployment" + snapshot: Final = _snapshot_model_cost_entries([deployment_id]) try: - _register_custom_pricing_for_request( - model="zai-org/GLM-5.2", - custom_llm_provider="openai", - kwargs={ - "input_cost_per_token": 0.00000096, - "output_cost_per_token": 0.00000302, - "metadata": {"model_info": {"id": deployment_id}}, - }, - model_info={"mode": "chat"}, + litellm.register_model({deployment_id: {**_AUDIO_RATES, "litellm_provider": "openai", "mode": "chat"}}) + response: Final = litellm.completion( + model="openai/gpt-audio-2025-08-28", + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + metadata={"model_info": {"id": deployment_id}}, + mock_response=_audio_reply(), + **_AUDIO_RATES, ) - - entry: Final = litellm.model_cost[deployment_id] - assert entry["input_cost_per_token"] == 0.00000096 - assert entry["output_cost_per_token"] == 0.00000302 - assert litellm.model_cost[builtin_key] == builtin_row_before + assert response._hidden_params["response_cost"] == pytest.approx(1000 * 4e-05 + 1000 * 8e-05) finally: - _restore_model_cost_entries(model_cost_entries) - litellm.open_ai_chat_completion_models.intersection_update(openai_models_before) + _restore_model_cost_entries(snapshot) + + +def test_direct_call_with_flat_rates_is_priced_from_them_without_writing_the_cost_map(): + model_key: Final = "openai/lit-direct-flat-rates-model" + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response=_audio_reply(), + input_cost_per_token=2.5e-06, + output_cost_per_token=1e-05, + ) + assert response._hidden_params["response_cost"] == pytest.approx(1000 * 2.5e-06 + 1000 * 1e-05) + assert model_key not in litellm.model_cost + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_direct_call_with_audio_rates_is_priced_per_modality_without_writing_the_cost_map(): + model_key: Final = "openai/lit-direct-audio-rates-model" + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response=_audio_reply(), + input_cost_per_token=2.5e-06, + output_cost_per_token=1e-05, + input_cost_per_audio_token=4e-05, + output_cost_per_audio_token=8e-05, + ) + assert response._hidden_params["response_cost"] == pytest.approx(1000 * 4e-05 + 1000 * 8e-05) + assert model_key not in litellm.model_cost + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_direct_call_with_directional_per_second_rates_is_priced_from_them_without_writing_the_cost_map(): + model_key: Final = "openai/lit-direct-per-second-model" + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + input_cost_per_second=0.004, + ) + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=0.004) == pytest.approx( + 0.004 * response._response_ms / 1000 + ) + assert response._hidden_params["response_cost"] > 0 + assert model_key not in litellm.model_cost + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_router_call_with_request_prices_bills_them_over_the_deployment_rates(): + from litellm import Router + + router: Final = Router( + model_list=[ + { + "model_name": "lit-request-override", + "litellm_params": { + "model": "openai/gpt-5.4-mini", + "api_key": "fake-key", + "mock_response": "ok", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "input_cost_per_audio_token": 4e-05, + "output_cost_per_audio_token": 8e-05, + }, + } + ] + ) + overridden: Final = router.completion( + model="lit-request-override", + messages=[{"role": "user", "content": "hello"}], + input_cost_per_token=5e-05, + output_cost_per_token=9e-05, + ) + assert overridden._hidden_params["response_cost"] == pytest.approx( + overridden.usage.prompt_tokens * 5e-05 + overridden.usage.completion_tokens * 9e-05 + ) + plain: Final = router.completion(model="lit-request-override", messages=[{"role": "user", "content": "hello"}]) + assert plain._hidden_params["response_cost"] == pytest.approx( + plain.usage.prompt_tokens * 1e-06 + plain.usage.completion_tokens * 2e-06 + ) + + +def test_direct_call_flat_rates_override_the_catalogs_tiers_per_direction(): + model_key: Final = "openai/lit-tiered-catalog-model" + litellm.register_model( + { + model_key: { + "litellm_provider": "openai", + "mode": "chat", + "tiered_pricing": [ + {"range": [0, 128000], "input_cost_per_token": 3e-07, "output_cost_per_token": 6e-07}, + {"range": [128000, 1000000], "input_cost_per_token": 6e-07, "output_cost_per_token": 1.2e-06}, + ], + } + }, + persist_across_reloads=False, + ) + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + input_cost_per_token=1e-05, + output_cost_per_token=2e-05, + ) + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * 1e-05 + response.usage.completion_tokens * 2e-05 + ) + input_only: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + input_cost_per_token=1e-05, + ) + assert input_only._hidden_params["response_cost"] == pytest.approx( + input_only.usage.prompt_tokens * 1e-05 + input_only.usage.completion_tokens * 6e-07 + ) + assert "input_cost_per_token" not in litellm.model_cost[model_key] + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_direct_call_with_only_a_per_second_rate_on_a_mapped_model_is_billed_per_second(): + model_key: Final = "openai/lit-mapped-token-rates-model" + litellm.register_model( + { + model_key: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + }, + persist_across_reloads=False, + ) + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + cost_per_second=0.02, + ) + token_rate_cost: Final = response.usage.prompt_tokens * 1e-06 + response.usage.completion_tokens * 2e-06 + assert response._hidden_params["response_cost"] != pytest.approx(token_rate_cost) + assert response._hidden_params["response_cost"] == pytest.approx(0.02 * response._response_ms / 1000, rel=0.25) + assert litellm.model_cost[model_key]["input_cost_per_token"] == 1e-06 + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_direct_call_flat_rate_applies_to_the_callers_own_tiers_not_the_catalogs(): + model_key: Final = "openai/lit-caller-tiers-model" + litellm.register_model( + { + model_key: { + "litellm_provider": "openai", + "mode": "chat", + "tiered_pricing": [ + {"range": [0, 1000000], "input_cost_per_token": 3e-07, "output_cost_per_token": 6e-07} + ], + } + }, + persist_across_reloads=False, + ) + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + input_cost_per_token=1e-05, + tiered_pricing=[{"range": [0, 1000000], "input_cost_per_token": 5e-06, "output_cost_per_token": 9e-06}], + ) + assert response._hidden_params["response_cost"] == pytest.approx( + response.usage.prompt_tokens * 1e-05 + response.usage.completion_tokens * 9e-06 + ) + assert litellm.model_cost[model_key]["tiered_pricing"][0]["output_cost_per_token"] == 6e-07 + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index aeb361d6944..7ab82422c5d 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2393,7 +2393,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None: @respx.mock @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") -def test_completion_registers_test_owned_model_prices() -> None: +def test_completion_custom_prices_price_the_call_without_writing_the_cost_map() -> None: model: Final = "migration-owned-pricing-model" messages: Final = [{"role": "user", "content": "price control"}] litellm.register_model( @@ -2436,8 +2436,9 @@ def test_completion_registers_test_owned_model_prices() -> None: ) assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages} - assert litellm.model_cost[model]["input_cost_per_token"] == 0.001 - assert litellm.model_cost[model]["output_cost_per_token"] == 0.002 + assert response._hidden_params["response_cost"] == pytest.approx(8 * 0.001 + 5 * 0.002) + assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001 + assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002 assert response._hidden_params["response_cost"] == pytest.approx(0.018)