diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 7ef5ce1d39b..330f643169d 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -600,8 +600,9 @@ class PrometheusLogger(CustomLogger): # Request queue time metric self.litellm_request_queue_time_metric = self._histogram_factory( "litellm_request_queue_time_seconds", - "Time (seconds) from request arrival at the proxy to the start of pre-call " - "processing -- includes authentication and any ASGI-level queueing", + "Time (seconds) the request waited between authentication completing and the " + "start of pre-call processing; excludes litellm_auth_latency and " + "litellm_proxy_pre_call_latency", labelnames=self.get_labels_for_metric("litellm_request_queue_time_seconds"), buckets=self.latency_buckets, ) @@ -2370,23 +2371,19 @@ class PrometheusLogger(CustomLogger): _labels, ) - # request queue time (time from arrival to processing start) -- read first so - # it can be folded into the total-latency metric below. start_time/end_time - # only span from after auth completes, so without this the "total" latency - # metric silently excludes auth and pre-call hook time. _litellm_params: Final = kwargs.get("litellm_params", {}) or {} - queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds") + _request_metadata: Final = _litellm_params.get("metadata") or {} + pre_processing_seconds: Final = _request_metadata.get("pre_processing_seconds") + queue_time_seconds: Final = _request_metadata.get("queue_time_seconds") - # total request latency: true end-to-end, from request arrival (queue_time_seconds, - # when available) through the end of processing. total_time_seconds: Final = self._safe_duration_seconds( start_time=start_time, end_time=end_time, ) if total_time_seconds is not None: _observed_total_time_seconds: Final = ( - total_time_seconds + queue_time_seconds - if queue_time_seconds is not None and queue_time_seconds >= 0 + total_time_seconds + pre_processing_seconds + if pre_processing_seconds is not None and pre_processing_seconds >= 0 else total_time_seconds ) _labels = prometheus_label_factory( diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 4cbd4213463..e00c499386c 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -3222,6 +3222,7 @@ async def user_api_key_auth( api_key=api_key, ) if recovered_auth_obj is not None: + request.state.litellm_auth_completed_at = datetime.now(timezone.utc) return recovered_auth_obj # Identity is now resolved. Seed it AFTER the auth span closes so the Baggage @@ -3253,6 +3254,7 @@ async def user_api_key_auth( except Exception as e: verbose_proxy_logger.warning("Principal projection at auth seam failed (non-fatal): %s", e) + request.state.litellm_auth_completed_at = datetime.now(timezone.utc) return user_api_key_auth_obj diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 2f39e6c71bc..9e0652cf43d 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1952,21 +1952,15 @@ class ProxyBaseLLMRequestProcessing: data=self.data, user_api_key_dict=user_api_key_dict, ) - # Calculate request queue time after add_litellm_data_to_request - # which sets arrival_time in proxy_server_request. Ends at start_time - # (not a freshly captured time.time() here) so this window is exactly - # [arrival_time, start_time], with zero overlap with the - # litellm_request_total_latency_metric window of [start_time, end_time] -- - # otherwise the few lines of add_litellm_data_to_request's own work would - # be double-counted across both metrics. proxy_server_request: Final = self.data.get("proxy_server_request", {}) arrival_time: Final = proxy_server_request.get("arrival_time") - queue_time_seconds = None - if arrival_time is not None: - queue_time_seconds = start_time.timestamp() - arrival_time + pre_processing_seconds: Final = start_time.timestamp() - arrival_time if arrival_time is not None else None + auth_completed_at: Final = getattr(request.state, "litellm_auth_completed_at", None) + queue_time_seconds: Final = ( + start_time.timestamp() - auth_completed_at.timestamp() if isinstance(auth_completed_at, datetime) else None + ) - # Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved - if queue_time_seconds is not None: + if pre_processing_seconds is not None or queue_time_seconds is not None: from litellm.proxy.litellm_pre_call_utils import _get_metadata_variable_name _metadata_variable_name: Final = _get_metadata_variable_name(request) @@ -1974,7 +1968,10 @@ class ProxyBaseLLMRequestProcessing: self.data[_metadata_variable_name] = {} if not isinstance(self.data[_metadata_variable_name], dict): self.data[_metadata_variable_name] = {} - self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds + if pre_processing_seconds is not None: + self.data[_metadata_variable_name]["pre_processing_seconds"] = pre_processing_seconds + if queue_time_seconds is not None: + self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds if isinstance(model, str): reject_url_valued_destination("model", model) diff --git a/tests/test_litellm/integrations/test_prometheus_queue_guardrail_metrics.py b/tests/test_litellm/integrations/test_prometheus_queue_guardrail_metrics.py index f04e8d0d2c7..b1d41b62dea 100644 --- a/tests/test_litellm/integrations/test_prometheus_queue_guardrail_metrics.py +++ b/tests/test_litellm/integrations/test_prometheus_queue_guardrail_metrics.py @@ -231,8 +231,9 @@ class TestPrometheusQueueTimeMetric: class TestPrometheusTotalLatencyMetric: """litellm_request_total_latency_metric must be true end-to-end latency: start_time - (set after auth already completed, see LIT-6012) plus queue_time_seconds (the - auth + pre-call setup window queue_time_seconds itself covers), not start_time alone.""" + (set after auth already completed, see LIT-6012) plus pre_processing_seconds (the + arrival-to-start window that includes auth), not start_time alone and not the + narrower post-auth queue_time_seconds (LIT-8007).""" @staticmethod def _enum_values() -> UserAPIKeyLabelValues: @@ -259,9 +260,10 @@ class TestPrometheusTotalLatencyMetric: route=None, ) - def test_total_latency_includes_queue_time_when_present(self): - """The observed total-latency value must be (end_time - start_time) + queue_time_seconds, - so auth/pre-call time (queue_time_seconds) is not silently excluded from "total" latency.""" + def test_total_latency_includes_pre_processing_time_not_queue_time(self): + """The observed total-latency value must be (end_time - start_time) + pre_processing_seconds, + so auth time is not silently excluded from "total" latency once queue_time_seconds no + longer covers it.""" prometheus_logger = PrometheusLogger() mock_metric = MagicMock() @@ -271,10 +273,16 @@ class TestPrometheusTotalLatencyMetric: start_time = datetime(2024, 1, 1, 0, 0, 0) end_time = datetime(2024, 1, 1, 0, 0, 2) # 2.0s of LLM-call/post-call time - queue_time_seconds = 0.5 # auth + pre-call setup time + pre_processing_seconds = 0.5 # arrival -> start_time, auth included + queue_time_seconds = 0.1 # auth completed -> start_time kwargs = { - "litellm_params": {"metadata": {"queue_time_seconds": queue_time_seconds}}, + "litellm_params": { + "metadata": { + "pre_processing_seconds": pre_processing_seconds, + "queue_time_seconds": queue_time_seconds, + } + }, "model": "gpt-3.5-turbo", "start_time": start_time, "end_time": end_time, @@ -293,8 +301,8 @@ class TestPrometheusTotalLatencyMetric: observed_value = mock_labeled_metric.observe.call_args_list[0][0][0] assert observed_value == pytest.approx(2.5) - def test_total_latency_falls_back_to_start_end_delta_without_queue_time(self): - """Without queue_time_seconds (e.g. a non-proxy caller), the metric must still + def test_total_latency_falls_back_to_start_end_delta_without_pre_processing_time(self): + """Without pre_processing_seconds (e.g. a non-proxy caller), the metric must still observe the plain end_time - start_time delta rather than erroring or dropping it.""" prometheus_logger = PrometheusLogger() @@ -326,8 +334,8 @@ class TestPrometheusTotalLatencyMetric: observed_value = mock_labeled_metric.observe.call_args_list[0][0][0] assert observed_value == pytest.approx(2.0) - def test_total_latency_ignores_negative_queue_time(self): - """A negative queue_time_seconds (clock skew / bad data) must not be added in -- + def test_total_latency_ignores_negative_pre_processing_time(self): + """A negative pre_processing_seconds (clock skew / bad data) must not be added in -- matches the existing >= 0 guard on the standalone queue-time metric.""" prometheus_logger = PrometheusLogger() @@ -340,7 +348,7 @@ class TestPrometheusTotalLatencyMetric: end_time = datetime(2024, 1, 1, 0, 0, 2) kwargs = { - "litellm_params": {"metadata": {"queue_time_seconds": -0.1}}, + "litellm_params": {"metadata": {"pre_processing_seconds": -0.1}}, "model": "gpt-3.5-turbo", "start_time": start_time, "end_time": end_time, diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index ba3e98ee718..94cc19b0688 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -7710,6 +7710,44 @@ class TestLitellmReceivedAtStamping: assert request.state.litellm_received_at == earlier +@pytest.mark.asyncio +async def test_user_api_key_auth_stamps_litellm_auth_completed_at_after_auth_work(): + """request.state.litellm_auth_completed_at is taken after auth finished, not at auth + entry, so a slow custom_auth handler cannot land in the queue-time window (LIT-8007).""" + from fastapi import Request + from starlette.datastructures import URL + + import litellm.proxy.proxy_server as _proxy_server_mod + + async def slow_custom_auth(request: Request, api_key: str) -> UserAPIKeyAuth: + await asyncio.sleep(0.05) + return UserAPIKeyAuth(api_key="sk-test", user_id="u1", user_role=LitellmUserRoles.PROXY_ADMIN) + + request = Request( + scope={ + "type": "http", + "headers": [(b"content-type", b"application/json")], + "method": "POST", + } + ) + request._url = URL(url="/chat/completions") + request._body = json.dumps({"model": "gpt-4o"}).encode() + + attrs = _proxy_server_attrs_for_custom_auth(user_custom_auth=slow_custom_auth) + originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} + try: + for k, v in attrs.items(): + setattr(_proxy_server_mod, k, v) + await user_api_key_auth(request=request, api_key="Bearer sk-test") + finally: + for k, v in originals.items(): + setattr(_proxy_server_mod, k, v) + + auth_completed_at = request.state.litellm_auth_completed_at + assert isinstance(auth_completed_at, datetime) + assert (auth_completed_at - request.state.litellm_received_at).total_seconds() >= 0.05 + + _RECORDING_DDTRACE = dedent( ''' import functools diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 4ac687625c2..00009948b95 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -1763,20 +1763,19 @@ class TestProxyBaseLLMRequestProcessing: assert headers["x-litellm-call-id"] == "explicit-call-id" - @pytest.mark.asyncio - async def test_queue_time_seconds_is_set_in_metadata(self, monkeypatch): - """ - Test that queue_time_seconds is correctly calculated and stored in metadata - after add_litellm_data_to_request populates arrival_time. - - This verifies the fix for the bug where queue_time_seconds was always None - because arrival_time was read BEFORE add_litellm_data_to_request set it. - """ + @staticmethod + async def _run_pre_call_logic_with_arrival( + monkeypatch: pytest.MonkeyPatch, + auth_completed_at: datetime.datetime | None, + ) -> tuple[dict, LiteLLMLoggingObj]: processing_obj = ProxyBaseLLMRequestProcessing(data={}) mock_request = MagicMock(spec=Request) mock_request.headers = {} mock_request.url = MagicMock() mock_request.url.path = "/v1/chat/completions" + mock_request.state = SimpleNamespace( + **({} if auth_completed_at is None else {"litellm_auth_completed_at": auth_completed_at}) + ) async def mock_add_litellm_data_to_request(*args, **kwargs): data = kwargs.get("data", args[0] if args else {}) @@ -1806,39 +1805,51 @@ class TestProxyBaseLLMRequestProcessing: mock_general_settings = {} mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) mock_proxy_config = MagicMock(spec=ProxyConfig) - route_type = "acompletion" - - ( - returned_data, - logging_obj, - ) = await processing_obj.common_processing_pre_call_logic( + return await processing_obj.common_processing_pre_call_logic( request=mock_request, general_settings=mock_general_settings, user_api_key_dict=mock_user_api_key_dict, proxy_logging_obj=mock_proxy_logging_obj, proxy_config=mock_proxy_config, - route_type=route_type, + route_type="acompletion", ) - # Verify queue_time_seconds is set and non-negative. Ends at start_time - # (captured before this mock runs, so it can precede the mock's own - # time.time() by a handful of microseconds) rather than a freshly - # captured time.time(), so a tiny tolerance below 0.5 is expected and - # correct -- see LIT-6012. - metadata = returned_data.get("metadata", {}) - assert "queue_time_seconds" in metadata, "queue_time_seconds should be set in metadata" - assert metadata["queue_time_seconds"] >= 0.49, ( - f"queue_time_seconds should be at least ~0.5, got {metadata['queue_time_seconds']}" + @pytest.mark.asyncio + async def test_queue_time_seconds_excludes_auth_and_pre_processing_covers_arrival(self, monkeypatch): + """ + Request arrived 0.5s ago and auth completed 0.2s ago (so auth took ~0.3s). + queue_time_seconds must span only [auth completed, start_time] so slow auth + (already measured by litellm_auth_latency) never lands in + litellm_request_queue_time_seconds, while pre_processing_seconds keeps the full + [arrival_time, start_time] window so total latency stays end to end (LIT-8007). + """ + auth_completed_at = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=0.2) + returned_data, logging_obj = await self._run_pre_call_logic_with_arrival( + monkeypatch, auth_completed_at=auth_completed_at ) - # queue_time_seconds must end exactly where logging_obj.start_time begins - # (the same start_time litellm_request_total_latency_metric's window - # starts from) so the two windows share a boundary, not an overlap. - # A mutant that reintroduces a separately-captured processing_start_time - # would make this assertion fail. + metadata = returned_data["metadata"] arrival_time = returned_data["proxy_server_request"]["arrival_time"] - assert arrival_time + metadata["queue_time_seconds"] == pytest.approx( - logging_obj.start_time.timestamp(), abs=1e-6 + processing_start = logging_obj.start_time.timestamp() + + assert metadata["pre_processing_seconds"] >= 0.49 + assert arrival_time + metadata["pre_processing_seconds"] == pytest.approx(processing_start, abs=1e-6) + + assert 0.19 <= metadata["queue_time_seconds"] < 0.45 + assert auth_completed_at.timestamp() + metadata["queue_time_seconds"] == pytest.approx( + processing_start, abs=1e-6 + ) + + @pytest.mark.asyncio + async def test_queue_time_seconds_omitted_without_auth_completion_stamp(self, monkeypatch): + """Without an auth completion stamp the queue window is unknown, so no queue_time_seconds + is fabricated from arrival_time, but pre_processing_seconds is still recorded.""" + returned_data, logging_obj = await self._run_pre_call_logic_with_arrival(monkeypatch, auth_completed_at=None) + + metadata = returned_data["metadata"] + assert "queue_time_seconds" not in metadata + assert returned_data["proxy_server_request"]["arrival_time"] + metadata["pre_processing_seconds"] == ( + pytest.approx(logging_obj.start_time.timestamp(), abs=1e-6) )