diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index 5ac2dca9ccf..61b77f8c811 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -182,6 +182,11 @@ def update_response_metadata( return metadata = ResponseMetadata(result) - metadata.set_hidden_params(logging_obj, model, kwargs) + # set_timing_metrics must run first: it assigns result._response_ms, which + # set_hidden_params reads (via _response_cost_calculator) to compute + # input_cost_per_second / output_cost_per_second pricing. Reversing this + # order makes per-second cost calculation read a missing/stale _response_ms + # and permanently cache a response_cost of 0.0 (see litellm issue #20228). metadata.set_timing_metrics(start_time, end_time, logging_obj) + metadata.set_hidden_params(logging_obj, model, kwargs) metadata.apply() diff --git a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py index 203c6d3da0d..7d026c1ce25 100644 --- a/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/test_litellm/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -6,8 +6,12 @@ through _hidden_params to the x-litellm-callback-duration-ms response header. """ import datetime +import uuid from unittest.mock import MagicMock +import pytest + +import litellm import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod import litellm.proxy.common_request_processing as common_request_processing_mod from litellm.litellm_core_utils.litellm_logging import Logging @@ -17,7 +21,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ) from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.utils import ModelResponse +from litellm.types.utils import ModelResponse, Usage class TestCallbackDurationMs: @@ -234,6 +238,67 @@ class TestDetailedTiming: assert "x-litellm-timing-pre-processing-ms" not in headers +class TestPerSecondCostUsesActualDuration: + """ + Regression test for https://github.com/BerriAI/litellm/issues/20228 as it + affects standard chat completions priced with input_cost_per_second. + + set_hidden_params triggers the real cost calculator, which for + input_cost_per_second models multiplies the rate by + completion_response._response_ms. That attribute is only assigned inside + set_timing_metrics, so calling set_hidden_params first makes the cost + calculator read a duration of 0 and the proxy permanently caches a + response_cost of 0.0 for that response. + """ + + def _make_logging_obj(self, model: str) -> Logging: + logging_obj = Logging( + model=model, + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + start_time=datetime.datetime.now(), + litellm_call_id=str(uuid.uuid4()), + function_id="1", + ) + logging_obj.update_environment_variables( + model=model, + optional_params={}, + litellm_params={}, + ) + return logging_obj + + def test_update_response_metadata_charges_for_actual_duration(self): + model = f"per-second-test-model-{uuid.uuid4()}" + litellm.model_cost[model] = { + "input_cost_per_second": 0.01, + "output_cost_per_token": 0, + "input_cost_per_token": 0, + "litellm_provider": "custom", + "mode": "chat", + } + try: + logging_obj = self._make_logging_obj(model) + result = ModelResponse(usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)) + + start = datetime.datetime(2025, 1, 1, 0, 0, 0) + end = datetime.datetime(2025, 1, 1, 0, 0, 2) # 2 second call + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model=model, + kwargs={}, + start_time=start, + end_time=end, + ) + + response_cost = result._hidden_params.get("response_cost") + assert response_cost == pytest.approx(0.02), f"expected 0.01/sec * 2 sec = 0.02, got {response_cost}" + finally: + litellm.model_cost.pop(model, None) + + class TestLoggingInitCallbackDuration: """Test that Logging.__init__ tracks deep copy time in callback_duration_ms."""