diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index e66be233be4..6a8d1dc7484 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -382,6 +382,14 @@ def _per_second_pricing_cost( model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model return None + return _per_second_pricing_cost_from_info(model_info, response_time_ms, audio_seconds) + + +def _per_second_pricing_cost_from_info( + model_info: ModelInfoBase, + response_time_ms: float | None, + audio_seconds: float = 0.0, +) -> tuple[float, float] | None: if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info): return None cost_per_second: Final = model_info.get("cost_per_second") @@ -404,7 +412,7 @@ def _per_second_pricing_cost( ) verbose_logger.debug( "For model=%s - cost_per_second: %s; response time: %s", - model, + model_info.get("key"), resolved_cost_per_second, response_time_ms, ) @@ -427,6 +435,7 @@ def cost_per_token( ### CUSTOM PRICING ### custom_cost_per_token: CostPerToken | None = None, custom_cost_per_second: float | None = None, + per_call_model_info: ModelInfo | None = None, ### NUMBER OF QUERIES ### number_of_queries: int | None = None, ### USAGE OBJECT ### @@ -538,6 +547,20 @@ def cost_per_token( if response_cost is not None: return response_cost[0], response_cost[1] + if per_call_model_info is not None: + per_second: Final = _per_second_pricing_cost_from_info( + per_call_model_info, response_time_ms, audio_transcription_file_duration + ) + if per_second is not None: + return per_second + return generic_cost_per_token( + model=model, + usage=usage_block, + custom_llm_provider=custom_llm_provider or "", + service_tier=service_tier, + model_info=per_call_model_info, + ) + model_cost_ref: Final = litellm.model_cost # Only callers that explicitly pass `custom_llm_provider` get the # dedup/prefix-join treatment. When provider is omitted, preserve legacy @@ -1668,6 +1691,7 @@ def completion_cost( ### CUSTOM PRICING ### custom_cost_per_token: CostPerToken | None = None, custom_cost_per_second: float | None = None, + per_call_model_info: ModelInfo | None = None, optional_params: dict | None = None, custom_pricing: bool | None = None, base_model: str | None = None, @@ -1721,6 +1745,7 @@ def completion_cost( quality=quality, n=n, custom_cost_per_token=custom_cost_per_token, + per_call_model_info=per_call_model_info, custom_cost_per_second=custom_cost_per_second, optional_params=optional_params, custom_pricing=custom_pricing, @@ -2206,6 +2231,7 @@ def completion_cost( region_name=None if explicit_pricing else region_name, custom_cost_per_second=custom_cost_per_second, custom_cost_per_token=custom_cost_per_token, + per_call_model_info=per_call_model_info, prompt_characters=prompt_characters, completion_characters=completion_characters, cache_creation_input_tokens=cache_creation_input_tokens, @@ -2438,6 +2464,7 @@ def response_cost_calculator( region_name: str | None = None, custom_cost_per_token: CostPerToken | None = None, custom_cost_per_second: float | None = None, + per_call_model_info: ModelInfo | None = None, ) -> float: """ Returns @@ -2494,6 +2521,7 @@ def response_cost_calculator( custom_pricing=custom_pricing, base_model=base_model, custom_cost_per_token=custom_cost_per_token, + per_call_model_info=per_call_model_info, custom_cost_per_second=custom_cost_per_second, prompt=prompt, standard_built_in_tools_params=standard_built_in_tools_params, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 639003c9a6a..d8418027ef7 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -706,6 +706,7 @@ class Logging(LiteLLMLoggingBaseClass): custom_pricing: bool = False custom_cost_per_token: CostPerToken | None = None custom_cost_per_second: float | None = None + custom_model_info: ModelInfo | None = None replaced_global_callback_keys: frozenset[str] = frozenset() stream_options = None litellm_request_debug: bool = False @@ -2205,6 +2206,7 @@ class Logging(LiteLLMLoggingBaseClass): "custom_pricing": custom_pricing, "custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None, "custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None, + "per_call_model_info": self.custom_model_info if custom_pricing else None, "prompt": prompt, "standard_built_in_tools_params": self.standard_built_in_tools_params, "router_model_id": router_model_id, @@ -2401,6 +2403,10 @@ class Logging(LiteLLMLoggingBaseClass): additional_headers=additional_headers, ) + def set_per_call_pricing(self, custom_model_info: ModelInfo | None) -> None: + """Price this call from its own full price entry instead of ``litellm.model_cost``.""" + self.custom_model_info = custom_model_info + def custom_pricing_for(self, result: object) -> bool: if self.spilled_over_ptu_for(result): return False diff --git a/litellm/main.py b/litellm/main.py index 2697eebf9c5..1c01c304c5f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -162,7 +162,8 @@ from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_s from litellm.types.utils import ( AgenticLoopParams, BaseUrlRoutingHint, - CostPerToken, + CustomPricingLiteLLMParams, + ModelInfo, ModelResponseStream, ProviderRoutingHints, RawRequestTypedDict, @@ -1421,21 +1422,54 @@ def _locals_snapshot(values: Mapping[str, object]) -> Mapping[str, object]: return values -_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken) +_METADATA_MODEL_INFO_ID: Final = TypeAdapter(Mapping[str, object]) +_MODEL_INFO: Final = TypeAdapter(ModelInfo) -def _per_call_cost_per_token(kwargs: Mapping[str, object]) -> CostPerToken | None: - """The flat per-token prices a caller put on this one call, priced from the logging object and never written - into ``litellm.model_cost``, so the next call for the same model name is priced from the cost map again.""" - if kwargs.get("input_cost_per_token") is None or kwargs.get("output_cost_per_token") is None: +def _is_router_call(kwargs: Mapping[str, object]) -> bool: + """A Router stamps ``model_info.id`` into the call's metadata and registers that deployment's full custom pricing + under that id when the deployment is added, so the cost calculator prices those calls from the registered entry.""" + for metadata_key in ("litellm_metadata", "metadata"): + metadata = kwargs.get(metadata_key) + if not isinstance(metadata, Mapping): + continue + model_info = _METADATA_MODEL_INFO_ID.validate_python(metadata).get("model_info") + if isinstance(model_info, Mapping) and model_info.get("id") is not None: + return True + return False + + +def _per_call_model_info( + model: str, custom_llm_provider: str, kwargs: Mapping[str, object], mode: Literal["chat", "embedding"] +) -> ModelInfo | None: + """The full price entry for one call: the cost map row for the model, if any, overlaid with every custom pricing + field the caller put on the call. It travels on the logging object and is never written into ``litellm.model_cost``, + so the next call for the same model name is priced from the cost map again.""" + custom: Final = { + field: kwargs[field] for field in CustomPricingLiteLLMParams.model_fields if kwargs.get(field) is not None + } + if not custom: + return None + unmapped: Final[Mapping[str, object]] = { + "key": model, + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + "input_cost_per_token": None, + "output_cost_per_token": None, + "litellm_provider": custom_llm_provider, + "mode": mode, + "supported_openai_params": None, + } + catalog: Final = _catalog_row(model, custom_llm_provider) or unmapped + return _MODEL_INFO.validate_python({**catalog, **custom}) + + +def _catalog_row(model: str, custom_llm_provider: str) -> ModelInfo | None: + try: + return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # get_model_info raises a bare Exception for an unmapped model return None - keys: Final = ( - "input_cost_per_token", - "output_cost_per_token", - "cache_read_input_token_cost", - "cache_creation_input_token_cost", - ) - return _COST_PER_TOKEN.validate_python({key: kwargs[key] for key in keys if kwargs.get(key) is not None}) def _record_agentic_loop_params( @@ -5902,8 +5936,8 @@ def completion( "aws_region_name", None ) # support region-based pricing for bedrock - logging.custom_cost_per_token = _per_call_cost_per_token(kwargs) - logging.custom_cost_per_second = cost_per_second + if not _is_router_call(kwargs): + logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "chat")) ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### custom_prompt_dict = {} if initial_prompt_value or roles or final_prompt_value or bos_token or eos_token: @@ -6910,7 +6944,6 @@ def embedding( headers.update(proxy_headers) except Exception as e: verbose_logger.warning("Failed to get proxy auth headers: %s", e) - cost_per_second: Final = kwargs.get("cost_per_second", None) openai_params: Final = [ "user", "dimensions", @@ -6958,8 +6991,8 @@ def embedding( ) logging: Final[LiteLLMLoggingObj] = litellm_logging_obj - logging.custom_cost_per_token = _per_call_cost_per_token(kwargs) - logging.custom_cost_per_second = cost_per_second + if not _is_router_call(kwargs): + logging.set_per_call_pricing(_per_call_model_info(model, custom_llm_provider, kwargs, "embedding")) logging.update_environment_variables( model=model, user=user, diff --git a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py index 5e40c2ebb60..4c253a10116 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_completion_handler.py @@ -549,7 +549,7 @@ async def test_completion_sagemaker( assert route.called assert response.choices[0].message.content == "SageMaker reply" - assert litellm.completion_cost(completion_response=response) == pytest.approx(rate) + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=rate) == pytest.approx(rate) @pytest.mark.asyncio diff --git a/tests/unit/test_register_model_custom_pricing.py b/tests/unit/test_register_model_custom_pricing.py index 13c68978656..78f2ed6581b 100644 --- a/tests/unit/test_register_model_custom_pricing.py +++ b/tests/unit/test_register_model_custom_pricing.py @@ -19,6 +19,14 @@ import pytest import litellm from litellm.litellm_core_utils.get_model_cost_map import _expand_model_aliases, adopt_model_cost_map +from litellm.types.utils import ( + Choices, + CompletionTokensDetailsWrapper, + Message, + ModelResponse, + PromptTokensDetailsWrapper, + Usage, +) from litellm.utils import _invalidate_model_cost_lowercase_map @@ -521,8 +529,8 @@ def test_embedding_router_zero_pricing_does_not_clobber_builtin_pricing(): def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): - """The request that carries custom pricing is costed with that pricing from its own logging object, while - ``litellm.model_cost`` keeps the built-in rate for siblings and gains no entry for the deployment.""" + """A Router registers the deployment's pricing under its id when the deployment is added. The routed request + is costed from that entry, and the shared backend key keeps the built-in rate for siblings.""" shared_key = "openai/text-embedding-3-small" deployment_id = "lit3991-wildcard-embed-custom" override_input_cost = 5e-05 @@ -531,6 +539,16 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): assert builtin_input_cost != override_input_cost try: + litellm.register_model( + { + deployment_id: { + "input_cost_per_token": override_input_cost, + "output_cost_per_token": override_input_cost * 2, + "litellm_provider": "openai", + "mode": "embedding", + } + } + ) response = litellm.embedding( model=shared_key, input=["hello"], @@ -545,7 +563,6 @@ def test_embedding_router_custom_pricing_costs_request_via_deployment_id(): assert response._hidden_params["response_cost"] == pytest.approx( response.usage.prompt_tokens * override_input_cost ) - assert deployment_id not in litellm.model_cost assert litellm.get_model_info(model=shared_key)["input_cost_per_token"] == builtin_input_cost finally: _restore_model_cost_entries(snapshot) @@ -1442,3 +1459,101 @@ def test_register_model_scopes_builtin_match_to_the_given_provider(): assert litellm.model_cost[builtin_key] == builtin_row_before finally: _restore_model_cost_entries(model_cost_entries) + + +def _audio_reply() -> ModelResponse: + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=1000, + total_tokens=2000, + prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=1000, text_tokens=0), + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=1000, text_tokens=0), + ) + return ModelResponse(model="gpt-audio-2025-08-28", choices=[Choices(message=Message(content="hi"))], usage=usage) + + +_AUDIO_RATES: Final = { + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1e-05, + "input_cost_per_audio_token": 4e-05, + "output_cost_per_audio_token": 8e-05, +} + + +def test_router_call_with_audio_rates_is_priced_per_modality_from_its_registered_deployment(): + """A Router registers the deployment's full custom pricing when the deployment is added. The per-call flat + prices must not override that entry, or 1000 audio input plus 1000 audio output tokens would bill at the text + rates ($0.0125) instead of the audio rates ($0.12).""" + deployment_id: Final = "audio-rates-deployment" + snapshot: Final = _snapshot_model_cost_entries([deployment_id]) + try: + litellm.register_model({deployment_id: {**_AUDIO_RATES, "litellm_provider": "openai", "mode": "chat"}}) + response: Final = litellm.completion( + model="openai/gpt-audio-2025-08-28", + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + metadata={"model_info": {"id": deployment_id}}, + mock_response=_audio_reply(), + **_AUDIO_RATES, + ) + assert response._hidden_params["response_cost"] == pytest.approx(1000 * 4e-05 + 1000 * 8e-05) + finally: + _restore_model_cost_entries(snapshot) + + +def test_direct_call_with_flat_rates_is_priced_from_them_without_writing_the_cost_map(): + model_key: Final = "openai/lit-direct-flat-rates-model" + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response=_audio_reply(), + input_cost_per_token=2.5e-06, + output_cost_per_token=1e-05, + ) + assert response._hidden_params["response_cost"] == pytest.approx(1000 * 2.5e-06 + 1000 * 1e-05) + assert model_key not in litellm.model_cost + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_direct_call_with_audio_rates_is_priced_per_modality_without_writing_the_cost_map(): + model_key: Final = "openai/lit-direct-audio-rates-model" + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response=_audio_reply(), + input_cost_per_token=2.5e-06, + output_cost_per_token=1e-05, + input_cost_per_audio_token=4e-05, + output_cost_per_audio_token=8e-05, + ) + assert response._hidden_params["response_cost"] == pytest.approx(1000 * 4e-05 + 1000 * 8e-05) + assert model_key not in litellm.model_cost + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() + + +def test_direct_call_with_directional_per_second_rates_is_priced_from_them_without_writing_the_cost_map(): + model_key: Final = "openai/lit-direct-per-second-model" + try: + response: Final = litellm.completion( + model=model_key, + messages=[{"role": "user", "content": "hello"}], + api_key="fake-key", + mock_response="ok", + input_cost_per_second=0.004, + ) + assert litellm.completion_cost(completion_response=response, custom_cost_per_second=0.004) == pytest.approx( + 0.004 * response._response_ms / 1000 + ) + assert response._hidden_params["response_cost"] == pytest.approx(0.004 * response._response_ms / 1000, rel=0.2) + assert model_key not in litellm.model_cost + finally: + litellm.model_cost.pop(model_key, None) + _invalidate_model_cost_lowercase_map() diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 5cf872f4ea7..bad829b82de 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2333,7 +2333,7 @@ def test_register_model_price_is_used_for_completion_cost() -> None: @respx.mock @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") -def test_completion_registers_test_owned_model_prices() -> None: +def test_completion_custom_prices_price_the_call_without_writing_the_cost_map() -> None: model: Final = "migration-owned-pricing-model" messages: Final = [{"role": "user", "content": "price control"}] litellm.register_model( @@ -2376,8 +2376,9 @@ def test_completion_registers_test_owned_model_prices() -> None: ) assert json.loads(route.calls[0].request.content) == {"model": model, "messages": messages} - assert litellm.model_cost[model]["input_cost_per_token"] == 0.001 - assert litellm.model_cost[model]["output_cost_per_token"] == 0.002 + assert response._hidden_params["response_cost"] == pytest.approx(8 * 0.001 + 5 * 0.002) + assert litellm.model_cost[model]["input_cost_per_token"] == 0.0001 + assert litellm.model_cost[model]["output_cost_per_token"] == 0.0002 assert response._hidden_params["response_cost"] == pytest.approx(0.018)