diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7e2c8d6e475..17324b8b3a3 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1785,18 +1785,7 @@ class Logging(LiteLLMLoggingBaseClass): router_model_id = self.get_router_model_id() ## RESPONSE COST ## - spilled_over: Final = is_spilled_over_ptu_request( - model_info=_deployment_model_info(self.litellm_params if hasattr(self, "litellm_params") else None), - response_headers=self.model_call_details.get("response_headers"), - additional_headers=result_additional_headers, - ) - custom_pricing: Final = ( - False - if spilled_over - else use_custom_pricing_for_model( - litellm_params=(self.litellm_params if hasattr(self, "litellm_params") else None) - ) - ) + custom_pricing: Final = self.custom_pricing_for(result) prompt = self._prompt_for_cost_calculation() @@ -1867,6 +1856,23 @@ class Logging(LiteLLMLoggingBaseClass): return None + def custom_pricing_for(self, result: object) -> bool: + litellm_params: Final = getattr(self, "litellm_params", None) + result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({}) + additional_headers: Final = ( + result_hidden_params.get("additional_headers") + if isinstance(result_hidden_params, dict) + else getattr(result_hidden_params, "additional_headers", None) + ) + spilled_over: Final = is_spilled_over_ptu_request( + model_info=_deployment_model_info(litellm_params), + response_headers=self.model_call_details.get("response_headers"), + additional_headers=additional_headers, + ) + return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params) + + + def _prompt_for_cost_calculation(self) -> str: """ The raw input string is only priced directly for text-to-speech, which bills per character. @@ -6349,7 +6355,7 @@ def get_standard_logging_object_payload( # Fall back to the actual deployment model set by the router in metadata. if base_model is None: base_model = metadata.get("deployment") - custom_pricing: Final = use_custom_pricing_for_model(litellm_params=litellm_params) + custom_pricing: Final = logging_obj.custom_pricing_for(init_response_obj) raw_response_cost: Final = kwargs.get("response_cost") llm_response_cost: Final[float] = raw_response_cost or 0.0 guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information")) diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 455bc2dd8c0..b9b4060d7b7 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -16,6 +16,7 @@ from openai._legacy_response import HttpxBinaryResponseContent import litellm from litellm._logging import session_id_var, trace_id_var +from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, SENTRY_DENYLIST, SENTRY_PII_DENYLIST from litellm.cost_calculator import ocr_batch_cost from litellm.integrations.custom_logger import CustomLogger @@ -7422,6 +7423,40 @@ class TestAzurePTUSpilloverCost: finally: self._unregister_models() + def test_spillover_payload_reports_the_pay_as_you_go_entry_billing_used(self, monkeypatch) -> None: + self._register_models() + try: + obj = self._logging_obj(dict(self.PTU_MODEL_INFO), flag="True", litellm_rate=0.0, monkeypatch=monkeypatch) + response = self._response() + response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"} + response_cost: Final = obj._response_cost_calculator(result=response) + + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": self.SERVED_MODEL, + "custom_llm_provider": "azure", + "litellm_params": obj.litellm_params, + "response_cost": response_cost, + }, + init_response_obj=response, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + logging_obj=obj, + status="success", + ) + + assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST) + assert payload is not None + model_map_value: Final = payload["model_map_information"]["model_map_value"] + assert model_map_value is not None + assert model_map_value["input_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["input_cost_per_token"] + assert model_map_value["output_cost_per_token"] == litellm.model_cost[self.SERVED_MODEL]["output_cost_per_token"] + assert response_cost == pytest.approx( + 100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"] + ) + finally: + self._unregister_models() + def test_spillover_header_does_not_touch_non_ptu_custom_pricing(self, monkeypatch) -> None: self._register_models() custom_model_id: Final = "non-ptu-custom-router-model-id"