diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 44c71a489d8..d99c236b4b3 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2047,20 +2047,23 @@ class Logging(LiteLLMLoggingBaseClass): usage=usage, ) - def custom_pricing_for(self, result: object) -> bool: - litellm_params: Final = getattr(self, "litellm_params", None) + def spilled_over_ptu_for(self, result: object) -> bool: result_hidden_params: Final = getattr(result, "_hidden_params", None) or MappingProxyType({}) additional_headers: Final = ( result_hidden_params.get("additional_headers") if isinstance(result_hidden_params, dict) else getattr(result_hidden_params, "additional_headers", None) ) - spilled_over: Final = is_spilled_over_ptu_request( - model_info=_deployment_model_info(litellm_params), + return is_spilled_over_ptu_request( + model_info=_deployment_model_info(getattr(self, "litellm_params", None)), response_headers=self.model_call_details.get("response_headers"), additional_headers=additional_headers, ) - return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params) + + def custom_pricing_for(self, result: object) -> bool: + if self.spilled_over_ptu_for(result): + return False + return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None)) def _prompt_for_cost_calculation(self) -> str: """ @@ -6617,13 +6620,16 @@ def get_standard_logging_object_payload( ) ## Get model cost information ## - base_model = _get_base_model_from_metadata(model_call_details=kwargs) - # The router overrides completion_response.model to the model-group alias before - # this payload is built, so cost-map lookup via that alias always misses. - # Fall back to the actual deployment model set by the router in metadata. - if base_model is None: - base_model = metadata.get("deployment") - custom_pricing: Final = logging_obj.custom_pricing_for(init_response_obj) + configured_base_model: Final = _get_base_model_from_metadata(model_call_details=kwargs) + spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj) + # The router overrides completion_response.model to the model-group alias before this payload is + # built, so cost-map lookup via that alias misses and the deployment model stands in for base_model. + # A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates, + # which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there. + base_model: Final = ( + metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model + ) + custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj) raw_response_cost: Final = kwargs.get("response_cost") llm_response_cost: Final[float] = raw_response_cost or 0.0 guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information")) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 7d9c33da923..1a94aebf4ea 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -4,6 +4,7 @@ from functools import reduce from typing import Final import httpx +from pydantic import TypeAdapter, ValidationError from litellm.constants import LITELLM_DETAILED_TIMING from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers @@ -101,6 +102,32 @@ def response_timing_metrics( return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms} +_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object]) + + +def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]: + try: + return _HIDDEN_PARAMS_DICT.validate_python(hidden_params) + except ValidationError: + return {} + + +def _existing_provider_response_model(result: object) -> str | None: + """The pricing model a response already carries, like the decisions path sets, wins over the response model.""" + hidden_params: Final[object] = getattr(result, "_hidden_params", None) + carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr( + hidden_params, "provider_response_model", None + ) + return carried if isinstance(carried, str) and carried else None + + +def _provider_response_model(result: object) -> str | None: + """The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the + client asked for and cost-map lookups for the standard logging payload would otherwise miss.""" + model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None + return model if isinstance(model, str) and model else None + + class ResponseMetadata: """ Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses @@ -133,6 +160,8 @@ class ResponseMetadata: preserve_litellm_internal_headers=True, ), "litellm_model_name": model, + "provider_response_model": _existing_provider_response_model(self.result) + or _provider_response_model(self.result), } self._update_hidden_params(new_params) diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py index 554447f4273..46837c6e4c3 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -17,6 +17,9 @@ import pytest import litellm import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod import litellm.proxy.common_request_processing as common_request_processing_mod +from litellm.cost_calculator import ( + _select_model_name_for_cost_calc, # pyright: ignore[reportPrivateUsage] # the line predates the public name +) from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ResponseMetadata, @@ -649,3 +652,38 @@ def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_ assert result._response_ms == pytest.approx(2000) assert result._hidden_params["response_cost"] == pytest.approx(0.02 * 2) + + +def test_update_response_metadata_keeps_the_provider_model_after_the_proxy_rewrites_response_model(): + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="azure/ptu-deployment", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="provider-model-response-metadata", + function_id="f", + ) + logging_obj.update_environment_variables( + model="azure/ptu-deployment", litellm_params={}, optional_params={}, custom_llm_provider="azure" + ) + result: Final = ModelResponse( + model="gpt-5.4-nano-2026-03-17", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model="azure/ptu-deployment", + kwargs={}, + start_time=start_time, + end_time=start_time + datetime.timedelta(seconds=2), + ) + result.model = "model-group-alias" + + assert ( + _select_model_name_for_cost_calc(model=None, completion_response=result, custom_llm_provider="azure") + == "azure/gpt-5.4-nano-2026-03-17" + ) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 2c83bb9bf2a..fd34cd391c5 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -8564,6 +8564,7 @@ class TestAzurePTUSpilloverCost: ROUTER_MODEL_ID: Final = "ptu-spill-router-model-id" SERVED_MODEL: Final = "azure/spill-served-model-ptu" + PTU_DEPLOYMENT: Final = "azure/spill-ptu-deployment" PTU_MODEL_INFO: Final = { "id": ROUTER_MODEL_ID, "team_id": "team-1", @@ -8599,7 +8600,9 @@ class TestAzurePTUSpilloverCost: litellm.model_cost.pop(TestAzurePTUSpilloverCost.ROUTER_MODEL_ID, None) litellm.model_cost.pop(TestAzurePTUSpilloverCost.SERVED_MODEL, None) - def _logging_obj(self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch) -> LitellmLogging: + def _logging_obj( + self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch, deployment: str | None = None + ) -> LitellmLogging: monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", flag) obj = LitellmLogging( model=self.SERVED_MODEL, @@ -8616,7 +8619,7 @@ class TestAzurePTUSpilloverCost: optional_params={}, litellm_params={ "api_base": "", - "metadata": {"model_info": model_info}, + "metadata": {"model_info": model_info, "deployment": deployment}, "input_cost_per_token": litellm_rate, "output_cost_per_token": litellm_rate, }, @@ -8720,7 +8723,6 @@ class TestAzurePTUSpilloverCost: finally: self._unregister_models() -||||||| parent of c6ae60c2eb (fixup! fix(logging): report the pay-as-you-go entry billing used on Azure PTU spillover (#47567)) def test_spillover_without_base_model_reports_the_pay_as_you_go_rates_billing_used(self, monkeypatch) -> None: self._register_models() litellm.model_cost[self.PTU_DEPLOYMENT] = {"litellm_provider": "azure", "mode": "chat"}