fix(prometheus): exclude auth and pre-call time from litellm_request_queue_time_seconds

Stamp request.state.litellm_auth_completed_at when user_api_key_auth returns and
measure queue_time_seconds from that point to the start of pre-call processing.
The full arrival-to-start window moves to pre_processing_seconds so
litellm_request_total_latency_metric stays end to end.

Resolves LIT-8007

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-17 17:37:24 +00:00
parent 11b99142eb
commit 0731500771
6 changed files with 121 additions and 68 deletions

View file

@ -600,8 +600,9 @@ class PrometheusLogger(CustomLogger):
# Request queue time metric
self.litellm_request_queue_time_metric = self._histogram_factory(
"litellm_request_queue_time_seconds",
"Time (seconds) from request arrival at the proxy to the start of pre-call "
"processing -- includes authentication and any ASGI-level queueing",
"Time (seconds) the request waited between authentication completing and the "
"start of pre-call processing; excludes litellm_auth_latency and "
"litellm_proxy_pre_call_latency",
labelnames=self.get_labels_for_metric("litellm_request_queue_time_seconds"),
buckets=self.latency_buckets,
)
@ -2370,23 +2371,19 @@ class PrometheusLogger(CustomLogger):
_labels,
)
# request queue time (time from arrival to processing start) -- read first so
# it can be folded into the total-latency metric below. start_time/end_time
# only span from after auth completes, so without this the "total" latency
# metric silently excludes auth and pre-call hook time.
_litellm_params: Final = kwargs.get("litellm_params", {}) or {}
queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
_request_metadata: Final = _litellm_params.get("metadata") or {}
pre_processing_seconds: Final = _request_metadata.get("pre_processing_seconds")
queue_time_seconds: Final = _request_metadata.get("queue_time_seconds")
# total request latency: true end-to-end, from request arrival (queue_time_seconds,
# when available) through the end of processing.
total_time_seconds: Final = self._safe_duration_seconds(
start_time=start_time,
end_time=end_time,
)
if total_time_seconds is not None:
_observed_total_time_seconds: Final = (
total_time_seconds + queue_time_seconds
if queue_time_seconds is not None and queue_time_seconds >= 0
total_time_seconds + pre_processing_seconds
if pre_processing_seconds is not None and pre_processing_seconds >= 0
else total_time_seconds
)
_labels = prometheus_label_factory(

View file

@ -3222,6 +3222,7 @@ async def user_api_key_auth(
api_key=api_key,
)
if recovered_auth_obj is not None:
request.state.litellm_auth_completed_at = datetime.now(timezone.utc)
return recovered_auth_obj
# Identity is now resolved. Seed it AFTER the auth span closes so the Baggage
@ -3253,6 +3254,7 @@ async def user_api_key_auth(
except Exception as e:
verbose_proxy_logger.warning("Principal projection at auth seam failed (non-fatal): %s", e)
request.state.litellm_auth_completed_at = datetime.now(timezone.utc)
return user_api_key_auth_obj

View file

@ -1952,21 +1952,15 @@ class ProxyBaseLLMRequestProcessing:
data=self.data,
user_api_key_dict=user_api_key_dict,
)
# Calculate request queue time after add_litellm_data_to_request
# which sets arrival_time in proxy_server_request. Ends at start_time
# (not a freshly captured time.time() here) so this window is exactly
# [arrival_time, start_time], with zero overlap with the
# litellm_request_total_latency_metric window of [start_time, end_time] --
# otherwise the few lines of add_litellm_data_to_request's own work would
# be double-counted across both metrics.
proxy_server_request: Final = self.data.get("proxy_server_request", {})
arrival_time: Final = proxy_server_request.get("arrival_time")
queue_time_seconds = None
if arrival_time is not None:
queue_time_seconds = start_time.timestamp() - arrival_time
pre_processing_seconds: Final = start_time.timestamp() - arrival_time if arrival_time is not None else None
auth_completed_at: Final = getattr(request.state, "litellm_auth_completed_at", None)
queue_time_seconds: Final = (
start_time.timestamp() - auth_completed_at.timestamp() if isinstance(auth_completed_at, datetime) else None
)
# Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved
if queue_time_seconds is not None:
if pre_processing_seconds is not None or queue_time_seconds is not None:
from litellm.proxy.litellm_pre_call_utils import _get_metadata_variable_name
_metadata_variable_name: Final = _get_metadata_variable_name(request)
@ -1974,7 +1968,10 @@ class ProxyBaseLLMRequestProcessing:
self.data[_metadata_variable_name] = {}
if not isinstance(self.data[_metadata_variable_name], dict):
self.data[_metadata_variable_name] = {}
self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds
if pre_processing_seconds is not None:
self.data[_metadata_variable_name]["pre_processing_seconds"] = pre_processing_seconds
if queue_time_seconds is not None:
self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds
if isinstance(model, str):
reject_url_valued_destination("model", model)

View file

@ -231,8 +231,9 @@ class TestPrometheusQueueTimeMetric:
class TestPrometheusTotalLatencyMetric:
"""litellm_request_total_latency_metric must be true end-to-end latency: start_time
(set after auth already completed, see LIT-6012) plus queue_time_seconds (the
auth + pre-call setup window queue_time_seconds itself covers), not start_time alone."""
(set after auth already completed, see LIT-6012) plus pre_processing_seconds (the
arrival-to-start window that includes auth), not start_time alone and not the
narrower post-auth queue_time_seconds (LIT-8007)."""
@staticmethod
def _enum_values() -> UserAPIKeyLabelValues:
@ -259,9 +260,10 @@ class TestPrometheusTotalLatencyMetric:
route=None,
)
def test_total_latency_includes_queue_time_when_present(self):
"""The observed total-latency value must be (end_time - start_time) + queue_time_seconds,
so auth/pre-call time (queue_time_seconds) is not silently excluded from "total" latency."""
def test_total_latency_includes_pre_processing_time_not_queue_time(self):
"""The observed total-latency value must be (end_time - start_time) + pre_processing_seconds,
so auth time is not silently excluded from "total" latency once queue_time_seconds no
longer covers it."""
prometheus_logger = PrometheusLogger()
mock_metric = MagicMock()
@ -271,10 +273,16 @@ class TestPrometheusTotalLatencyMetric:
start_time = datetime(2024, 1, 1, 0, 0, 0)
end_time = datetime(2024, 1, 1, 0, 0, 2) # 2.0s of LLM-call/post-call time
queue_time_seconds = 0.5 # auth + pre-call setup time
pre_processing_seconds = 0.5 # arrival -> start_time, auth included
queue_time_seconds = 0.1 # auth completed -> start_time
kwargs = {
"litellm_params": {"metadata": {"queue_time_seconds": queue_time_seconds}},
"litellm_params": {
"metadata": {
"pre_processing_seconds": pre_processing_seconds,
"queue_time_seconds": queue_time_seconds,
}
},
"model": "gpt-3.5-turbo",
"start_time": start_time,
"end_time": end_time,
@ -293,8 +301,8 @@ class TestPrometheusTotalLatencyMetric:
observed_value = mock_labeled_metric.observe.call_args_list[0][0][0]
assert observed_value == pytest.approx(2.5)
def test_total_latency_falls_back_to_start_end_delta_without_queue_time(self):
"""Without queue_time_seconds (e.g. a non-proxy caller), the metric must still
def test_total_latency_falls_back_to_start_end_delta_without_pre_processing_time(self):
"""Without pre_processing_seconds (e.g. a non-proxy caller), the metric must still
observe the plain end_time - start_time delta rather than erroring or dropping it."""
prometheus_logger = PrometheusLogger()
@ -326,8 +334,8 @@ class TestPrometheusTotalLatencyMetric:
observed_value = mock_labeled_metric.observe.call_args_list[0][0][0]
assert observed_value == pytest.approx(2.0)
def test_total_latency_ignores_negative_queue_time(self):
"""A negative queue_time_seconds (clock skew / bad data) must not be added in --
def test_total_latency_ignores_negative_pre_processing_time(self):
"""A negative pre_processing_seconds (clock skew / bad data) must not be added in --
matches the existing >= 0 guard on the standalone queue-time metric."""
prometheus_logger = PrometheusLogger()
@ -340,7 +348,7 @@ class TestPrometheusTotalLatencyMetric:
end_time = datetime(2024, 1, 1, 0, 0, 2)
kwargs = {
"litellm_params": {"metadata": {"queue_time_seconds": -0.1}},
"litellm_params": {"metadata": {"pre_processing_seconds": -0.1}},
"model": "gpt-3.5-turbo",
"start_time": start_time,
"end_time": end_time,

View file

@ -7710,6 +7710,44 @@ class TestLitellmReceivedAtStamping:
assert request.state.litellm_received_at == earlier
@pytest.mark.asyncio
async def test_user_api_key_auth_stamps_litellm_auth_completed_at_after_auth_work():
"""request.state.litellm_auth_completed_at is taken after auth finished, not at auth
entry, so a slow custom_auth handler cannot land in the queue-time window (LIT-8007)."""
from fastapi import Request
from starlette.datastructures import URL
import litellm.proxy.proxy_server as _proxy_server_mod
async def slow_custom_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
await asyncio.sleep(0.05)
return UserAPIKeyAuth(api_key="sk-test", user_id="u1", user_role=LitellmUserRoles.PROXY_ADMIN)
request = Request(
scope={
"type": "http",
"headers": [(b"content-type", b"application/json")],
"method": "POST",
}
)
request._url = URL(url="/chat/completions")
request._body = json.dumps({"model": "gpt-4o"}).encode()
attrs = _proxy_server_attrs_for_custom_auth(user_custom_auth=slow_custom_auth)
originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs}
try:
for k, v in attrs.items():
setattr(_proxy_server_mod, k, v)
await user_api_key_auth(request=request, api_key="Bearer sk-test")
finally:
for k, v in originals.items():
setattr(_proxy_server_mod, k, v)
auth_completed_at = request.state.litellm_auth_completed_at
assert isinstance(auth_completed_at, datetime)
assert (auth_completed_at - request.state.litellm_received_at).total_seconds() >= 0.05
_RECORDING_DDTRACE = dedent(
'''
import functools

View file

@ -1763,20 +1763,19 @@ class TestProxyBaseLLMRequestProcessing:
assert headers["x-litellm-call-id"] == "explicit-call-id"
@pytest.mark.asyncio
async def test_queue_time_seconds_is_set_in_metadata(self, monkeypatch):
"""
Test that queue_time_seconds is correctly calculated and stored in metadata
after add_litellm_data_to_request populates arrival_time.
This verifies the fix for the bug where queue_time_seconds was always None
because arrival_time was read BEFORE add_litellm_data_to_request set it.
"""
@staticmethod
async def _run_pre_call_logic_with_arrival(
monkeypatch: pytest.MonkeyPatch,
auth_completed_at: datetime.datetime | None,
) -> tuple[dict, LiteLLMLoggingObj]:
processing_obj = ProxyBaseLLMRequestProcessing(data={})
mock_request = MagicMock(spec=Request)
mock_request.headers = {}
mock_request.url = MagicMock()
mock_request.url.path = "/v1/chat/completions"
mock_request.state = SimpleNamespace(
**({} if auth_completed_at is None else {"litellm_auth_completed_at": auth_completed_at})
)
async def mock_add_litellm_data_to_request(*args, **kwargs):
data = kwargs.get("data", args[0] if args else {})
@ -1806,39 +1805,51 @@ class TestProxyBaseLLMRequestProcessing:
mock_general_settings = {}
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
mock_proxy_config = MagicMock(spec=ProxyConfig)
route_type = "acompletion"
(
returned_data,
logging_obj,
) = await processing_obj.common_processing_pre_call_logic(
return await processing_obj.common_processing_pre_call_logic(
request=mock_request,
general_settings=mock_general_settings,
user_api_key_dict=mock_user_api_key_dict,
proxy_logging_obj=mock_proxy_logging_obj,
proxy_config=mock_proxy_config,
route_type=route_type,
route_type="acompletion",
)
# Verify queue_time_seconds is set and non-negative. Ends at start_time
# (captured before this mock runs, so it can precede the mock's own
# time.time() by a handful of microseconds) rather than a freshly
# captured time.time(), so a tiny tolerance below 0.5 is expected and
# correct -- see LIT-6012.
metadata = returned_data.get("metadata", {})
assert "queue_time_seconds" in metadata, "queue_time_seconds should be set in metadata"
assert metadata["queue_time_seconds"] >= 0.49, (
f"queue_time_seconds should be at least ~0.5, got {metadata['queue_time_seconds']}"
@pytest.mark.asyncio
async def test_queue_time_seconds_excludes_auth_and_pre_processing_covers_arrival(self, monkeypatch):
"""
Request arrived 0.5s ago and auth completed 0.2s ago (so auth took ~0.3s).
queue_time_seconds must span only [auth completed, start_time] so slow auth
(already measured by litellm_auth_latency) never lands in
litellm_request_queue_time_seconds, while pre_processing_seconds keeps the full
[arrival_time, start_time] window so total latency stays end to end (LIT-8007).
"""
auth_completed_at = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=0.2)
returned_data, logging_obj = await self._run_pre_call_logic_with_arrival(
monkeypatch, auth_completed_at=auth_completed_at
)
# queue_time_seconds must end exactly where logging_obj.start_time begins
# (the same start_time litellm_request_total_latency_metric's window
# starts from) so the two windows share a boundary, not an overlap.
# A mutant that reintroduces a separately-captured processing_start_time
# would make this assertion fail.
metadata = returned_data["metadata"]
arrival_time = returned_data["proxy_server_request"]["arrival_time"]
assert arrival_time + metadata["queue_time_seconds"] == pytest.approx(
logging_obj.start_time.timestamp(), abs=1e-6
processing_start = logging_obj.start_time.timestamp()
assert metadata["pre_processing_seconds"] >= 0.49
assert arrival_time + metadata["pre_processing_seconds"] == pytest.approx(processing_start, abs=1e-6)
assert 0.19 <= metadata["queue_time_seconds"] < 0.45
assert auth_completed_at.timestamp() + metadata["queue_time_seconds"] == pytest.approx(
processing_start, abs=1e-6
)
@pytest.mark.asyncio
async def test_queue_time_seconds_omitted_without_auth_completion_stamp(self, monkeypatch):
"""Without an auth completion stamp the queue window is unknown, so no queue_time_seconds
is fabricated from arrival_time, but pre_processing_seconds is still recorded."""
returned_data, logging_obj = await self._run_pre_call_logic_with_arrival(monkeypatch, auth_completed_at=None)
metadata = returned_data["metadata"]
assert "queue_time_seconds" not in metadata
assert returned_data["proxy_server_request"]["arrival_time"] + metadata["pre_processing_seconds"] == (
pytest.approx(logging_obj.start_time.timestamp(), abs=1e-6)
)