mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(prometheus): exclude auth and pre-call time from litellm_request_queue_time_seconds
Stamp request.state.litellm_auth_completed_at when user_api_key_auth returns and measure queue_time_seconds from that point to the start of pre-call processing. The full arrival-to-start window moves to pre_processing_seconds so litellm_request_total_latency_metric stays end to end. Resolves LIT-8007 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
11b99142eb
commit
0731500771
6 changed files with 121 additions and 68 deletions
|
|
@ -600,8 +600,9 @@ class PrometheusLogger(CustomLogger):
|
|||
# Request queue time metric
|
||||
self.litellm_request_queue_time_metric = self._histogram_factory(
|
||||
"litellm_request_queue_time_seconds",
|
||||
"Time (seconds) from request arrival at the proxy to the start of pre-call "
|
||||
"processing -- includes authentication and any ASGI-level queueing",
|
||||
"Time (seconds) the request waited between authentication completing and the "
|
||||
"start of pre-call processing; excludes litellm_auth_latency and "
|
||||
"litellm_proxy_pre_call_latency",
|
||||
labelnames=self.get_labels_for_metric("litellm_request_queue_time_seconds"),
|
||||
buckets=self.latency_buckets,
|
||||
)
|
||||
|
|
@ -2370,23 +2371,19 @@ class PrometheusLogger(CustomLogger):
|
|||
_labels,
|
||||
)
|
||||
|
||||
# request queue time (time from arrival to processing start) -- read first so
|
||||
# it can be folded into the total-latency metric below. start_time/end_time
|
||||
# only span from after auth completes, so without this the "total" latency
|
||||
# metric silently excludes auth and pre-call hook time.
|
||||
_litellm_params: Final = kwargs.get("litellm_params", {}) or {}
|
||||
queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
|
||||
_request_metadata: Final = _litellm_params.get("metadata") or {}
|
||||
pre_processing_seconds: Final = _request_metadata.get("pre_processing_seconds")
|
||||
queue_time_seconds: Final = _request_metadata.get("queue_time_seconds")
|
||||
|
||||
# total request latency: true end-to-end, from request arrival (queue_time_seconds,
|
||||
# when available) through the end of processing.
|
||||
total_time_seconds: Final = self._safe_duration_seconds(
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
if total_time_seconds is not None:
|
||||
_observed_total_time_seconds: Final = (
|
||||
total_time_seconds + queue_time_seconds
|
||||
if queue_time_seconds is not None and queue_time_seconds >= 0
|
||||
total_time_seconds + pre_processing_seconds
|
||||
if pre_processing_seconds is not None and pre_processing_seconds >= 0
|
||||
else total_time_seconds
|
||||
)
|
||||
_labels = prometheus_label_factory(
|
||||
|
|
|
|||
|
|
@ -3222,6 +3222,7 @@ async def user_api_key_auth(
|
|||
api_key=api_key,
|
||||
)
|
||||
if recovered_auth_obj is not None:
|
||||
request.state.litellm_auth_completed_at = datetime.now(timezone.utc)
|
||||
return recovered_auth_obj
|
||||
|
||||
# Identity is now resolved. Seed it AFTER the auth span closes so the Baggage
|
||||
|
|
@ -3253,6 +3254,7 @@ async def user_api_key_auth(
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.warning("Principal projection at auth seam failed (non-fatal): %s", e)
|
||||
|
||||
request.state.litellm_auth_completed_at = datetime.now(timezone.utc)
|
||||
return user_api_key_auth_obj
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1952,21 +1952,15 @@ class ProxyBaseLLMRequestProcessing:
|
|||
data=self.data,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
# Calculate request queue time after add_litellm_data_to_request
|
||||
# which sets arrival_time in proxy_server_request. Ends at start_time
|
||||
# (not a freshly captured time.time() here) so this window is exactly
|
||||
# [arrival_time, start_time], with zero overlap with the
|
||||
# litellm_request_total_latency_metric window of [start_time, end_time] --
|
||||
# otherwise the few lines of add_litellm_data_to_request's own work would
|
||||
# be double-counted across both metrics.
|
||||
proxy_server_request: Final = self.data.get("proxy_server_request", {})
|
||||
arrival_time: Final = proxy_server_request.get("arrival_time")
|
||||
queue_time_seconds = None
|
||||
if arrival_time is not None:
|
||||
queue_time_seconds = start_time.timestamp() - arrival_time
|
||||
pre_processing_seconds: Final = start_time.timestamp() - arrival_time if arrival_time is not None else None
|
||||
auth_completed_at: Final = getattr(request.state, "litellm_auth_completed_at", None)
|
||||
queue_time_seconds: Final = (
|
||||
start_time.timestamp() - auth_completed_at.timestamp() if isinstance(auth_completed_at, datetime) else None
|
||||
)
|
||||
|
||||
# Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved
|
||||
if queue_time_seconds is not None:
|
||||
if pre_processing_seconds is not None or queue_time_seconds is not None:
|
||||
from litellm.proxy.litellm_pre_call_utils import _get_metadata_variable_name
|
||||
|
||||
_metadata_variable_name: Final = _get_metadata_variable_name(request)
|
||||
|
|
@ -1974,7 +1968,10 @@ class ProxyBaseLLMRequestProcessing:
|
|||
self.data[_metadata_variable_name] = {}
|
||||
if not isinstance(self.data[_metadata_variable_name], dict):
|
||||
self.data[_metadata_variable_name] = {}
|
||||
self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds
|
||||
if pre_processing_seconds is not None:
|
||||
self.data[_metadata_variable_name]["pre_processing_seconds"] = pre_processing_seconds
|
||||
if queue_time_seconds is not None:
|
||||
self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds
|
||||
|
||||
if isinstance(model, str):
|
||||
reject_url_valued_destination("model", model)
|
||||
|
|
|
|||
|
|
@ -231,8 +231,9 @@ class TestPrometheusQueueTimeMetric:
|
|||
|
||||
class TestPrometheusTotalLatencyMetric:
|
||||
"""litellm_request_total_latency_metric must be true end-to-end latency: start_time
|
||||
(set after auth already completed, see LIT-6012) plus queue_time_seconds (the
|
||||
auth + pre-call setup window queue_time_seconds itself covers), not start_time alone."""
|
||||
(set after auth already completed, see LIT-6012) plus pre_processing_seconds (the
|
||||
arrival-to-start window that includes auth), not start_time alone and not the
|
||||
narrower post-auth queue_time_seconds (LIT-8007)."""
|
||||
|
||||
@staticmethod
|
||||
def _enum_values() -> UserAPIKeyLabelValues:
|
||||
|
|
@ -259,9 +260,10 @@ class TestPrometheusTotalLatencyMetric:
|
|||
route=None,
|
||||
)
|
||||
|
||||
def test_total_latency_includes_queue_time_when_present(self):
|
||||
"""The observed total-latency value must be (end_time - start_time) + queue_time_seconds,
|
||||
so auth/pre-call time (queue_time_seconds) is not silently excluded from "total" latency."""
|
||||
def test_total_latency_includes_pre_processing_time_not_queue_time(self):
|
||||
"""The observed total-latency value must be (end_time - start_time) + pre_processing_seconds,
|
||||
so auth time is not silently excluded from "total" latency once queue_time_seconds no
|
||||
longer covers it."""
|
||||
prometheus_logger = PrometheusLogger()
|
||||
|
||||
mock_metric = MagicMock()
|
||||
|
|
@ -271,10 +273,16 @@ class TestPrometheusTotalLatencyMetric:
|
|||
|
||||
start_time = datetime(2024, 1, 1, 0, 0, 0)
|
||||
end_time = datetime(2024, 1, 1, 0, 0, 2) # 2.0s of LLM-call/post-call time
|
||||
queue_time_seconds = 0.5 # auth + pre-call setup time
|
||||
pre_processing_seconds = 0.5 # arrival -> start_time, auth included
|
||||
queue_time_seconds = 0.1 # auth completed -> start_time
|
||||
|
||||
kwargs = {
|
||||
"litellm_params": {"metadata": {"queue_time_seconds": queue_time_seconds}},
|
||||
"litellm_params": {
|
||||
"metadata": {
|
||||
"pre_processing_seconds": pre_processing_seconds,
|
||||
"queue_time_seconds": queue_time_seconds,
|
||||
}
|
||||
},
|
||||
"model": "gpt-3.5-turbo",
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
|
|
@ -293,8 +301,8 @@ class TestPrometheusTotalLatencyMetric:
|
|||
observed_value = mock_labeled_metric.observe.call_args_list[0][0][0]
|
||||
assert observed_value == pytest.approx(2.5)
|
||||
|
||||
def test_total_latency_falls_back_to_start_end_delta_without_queue_time(self):
|
||||
"""Without queue_time_seconds (e.g. a non-proxy caller), the metric must still
|
||||
def test_total_latency_falls_back_to_start_end_delta_without_pre_processing_time(self):
|
||||
"""Without pre_processing_seconds (e.g. a non-proxy caller), the metric must still
|
||||
observe the plain end_time - start_time delta rather than erroring or dropping it."""
|
||||
prometheus_logger = PrometheusLogger()
|
||||
|
||||
|
|
@ -326,8 +334,8 @@ class TestPrometheusTotalLatencyMetric:
|
|||
observed_value = mock_labeled_metric.observe.call_args_list[0][0][0]
|
||||
assert observed_value == pytest.approx(2.0)
|
||||
|
||||
def test_total_latency_ignores_negative_queue_time(self):
|
||||
"""A negative queue_time_seconds (clock skew / bad data) must not be added in --
|
||||
def test_total_latency_ignores_negative_pre_processing_time(self):
|
||||
"""A negative pre_processing_seconds (clock skew / bad data) must not be added in --
|
||||
matches the existing >= 0 guard on the standalone queue-time metric."""
|
||||
prometheus_logger = PrometheusLogger()
|
||||
|
||||
|
|
@ -340,7 +348,7 @@ class TestPrometheusTotalLatencyMetric:
|
|||
end_time = datetime(2024, 1, 1, 0, 0, 2)
|
||||
|
||||
kwargs = {
|
||||
"litellm_params": {"metadata": {"queue_time_seconds": -0.1}},
|
||||
"litellm_params": {"metadata": {"pre_processing_seconds": -0.1}},
|
||||
"model": "gpt-3.5-turbo",
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
|
|
|
|||
|
|
@ -7710,6 +7710,44 @@ class TestLitellmReceivedAtStamping:
|
|||
assert request.state.litellm_received_at == earlier
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_api_key_auth_stamps_litellm_auth_completed_at_after_auth_work():
|
||||
"""request.state.litellm_auth_completed_at is taken after auth finished, not at auth
|
||||
entry, so a slow custom_auth handler cannot land in the queue-time window (LIT-8007)."""
|
||||
from fastapi import Request
|
||||
from starlette.datastructures import URL
|
||||
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
|
||||
async def slow_custom_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
|
||||
await asyncio.sleep(0.05)
|
||||
return UserAPIKeyAuth(api_key="sk-test", user_id="u1", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
||||
request = Request(
|
||||
scope={
|
||||
"type": "http",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
"method": "POST",
|
||||
}
|
||||
)
|
||||
request._url = URL(url="/chat/completions")
|
||||
request._body = json.dumps({"model": "gpt-4o"}).encode()
|
||||
|
||||
attrs = _proxy_server_attrs_for_custom_auth(user_custom_auth=slow_custom_auth)
|
||||
originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs}
|
||||
try:
|
||||
for k, v in attrs.items():
|
||||
setattr(_proxy_server_mod, k, v)
|
||||
await user_api_key_auth(request=request, api_key="Bearer sk-test")
|
||||
finally:
|
||||
for k, v in originals.items():
|
||||
setattr(_proxy_server_mod, k, v)
|
||||
|
||||
auth_completed_at = request.state.litellm_auth_completed_at
|
||||
assert isinstance(auth_completed_at, datetime)
|
||||
assert (auth_completed_at - request.state.litellm_received_at).total_seconds() >= 0.05
|
||||
|
||||
|
||||
_RECORDING_DDTRACE = dedent(
|
||||
'''
|
||||
import functools
|
||||
|
|
|
|||
|
|
@ -1763,20 +1763,19 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
|
||||
assert headers["x-litellm-call-id"] == "explicit-call-id"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_time_seconds_is_set_in_metadata(self, monkeypatch):
|
||||
"""
|
||||
Test that queue_time_seconds is correctly calculated and stored in metadata
|
||||
after add_litellm_data_to_request populates arrival_time.
|
||||
|
||||
This verifies the fix for the bug where queue_time_seconds was always None
|
||||
because arrival_time was read BEFORE add_litellm_data_to_request set it.
|
||||
"""
|
||||
@staticmethod
|
||||
async def _run_pre_call_logic_with_arrival(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
auth_completed_at: datetime.datetime | None,
|
||||
) -> tuple[dict, LiteLLMLoggingObj]:
|
||||
processing_obj = ProxyBaseLLMRequestProcessing(data={})
|
||||
mock_request = MagicMock(spec=Request)
|
||||
mock_request.headers = {}
|
||||
mock_request.url = MagicMock()
|
||||
mock_request.url.path = "/v1/chat/completions"
|
||||
mock_request.state = SimpleNamespace(
|
||||
**({} if auth_completed_at is None else {"litellm_auth_completed_at": auth_completed_at})
|
||||
)
|
||||
|
||||
async def mock_add_litellm_data_to_request(*args, **kwargs):
|
||||
data = kwargs.get("data", args[0] if args else {})
|
||||
|
|
@ -1806,39 +1805,51 @@ class TestProxyBaseLLMRequestProcessing:
|
|||
mock_general_settings = {}
|
||||
mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth)
|
||||
mock_proxy_config = MagicMock(spec=ProxyConfig)
|
||||
route_type = "acompletion"
|
||||
|
||||
(
|
||||
returned_data,
|
||||
logging_obj,
|
||||
) = await processing_obj.common_processing_pre_call_logic(
|
||||
return await processing_obj.common_processing_pre_call_logic(
|
||||
request=mock_request,
|
||||
general_settings=mock_general_settings,
|
||||
user_api_key_dict=mock_user_api_key_dict,
|
||||
proxy_logging_obj=mock_proxy_logging_obj,
|
||||
proxy_config=mock_proxy_config,
|
||||
route_type=route_type,
|
||||
route_type="acompletion",
|
||||
)
|
||||
|
||||
# Verify queue_time_seconds is set and non-negative. Ends at start_time
|
||||
# (captured before this mock runs, so it can precede the mock's own
|
||||
# time.time() by a handful of microseconds) rather than a freshly
|
||||
# captured time.time(), so a tiny tolerance below 0.5 is expected and
|
||||
# correct -- see LIT-6012.
|
||||
metadata = returned_data.get("metadata", {})
|
||||
assert "queue_time_seconds" in metadata, "queue_time_seconds should be set in metadata"
|
||||
assert metadata["queue_time_seconds"] >= 0.49, (
|
||||
f"queue_time_seconds should be at least ~0.5, got {metadata['queue_time_seconds']}"
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_time_seconds_excludes_auth_and_pre_processing_covers_arrival(self, monkeypatch):
|
||||
"""
|
||||
Request arrived 0.5s ago and auth completed 0.2s ago (so auth took ~0.3s).
|
||||
queue_time_seconds must span only [auth completed, start_time] so slow auth
|
||||
(already measured by litellm_auth_latency) never lands in
|
||||
litellm_request_queue_time_seconds, while pre_processing_seconds keeps the full
|
||||
[arrival_time, start_time] window so total latency stays end to end (LIT-8007).
|
||||
"""
|
||||
auth_completed_at = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=0.2)
|
||||
returned_data, logging_obj = await self._run_pre_call_logic_with_arrival(
|
||||
monkeypatch, auth_completed_at=auth_completed_at
|
||||
)
|
||||
|
||||
# queue_time_seconds must end exactly where logging_obj.start_time begins
|
||||
# (the same start_time litellm_request_total_latency_metric's window
|
||||
# starts from) so the two windows share a boundary, not an overlap.
|
||||
# A mutant that reintroduces a separately-captured processing_start_time
|
||||
# would make this assertion fail.
|
||||
metadata = returned_data["metadata"]
|
||||
arrival_time = returned_data["proxy_server_request"]["arrival_time"]
|
||||
assert arrival_time + metadata["queue_time_seconds"] == pytest.approx(
|
||||
logging_obj.start_time.timestamp(), abs=1e-6
|
||||
processing_start = logging_obj.start_time.timestamp()
|
||||
|
||||
assert metadata["pre_processing_seconds"] >= 0.49
|
||||
assert arrival_time + metadata["pre_processing_seconds"] == pytest.approx(processing_start, abs=1e-6)
|
||||
|
||||
assert 0.19 <= metadata["queue_time_seconds"] < 0.45
|
||||
assert auth_completed_at.timestamp() + metadata["queue_time_seconds"] == pytest.approx(
|
||||
processing_start, abs=1e-6
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_time_seconds_omitted_without_auth_completion_stamp(self, monkeypatch):
|
||||
"""Without an auth completion stamp the queue window is unknown, so no queue_time_seconds
|
||||
is fabricated from arrival_time, but pre_processing_seconds is still recorded."""
|
||||
returned_data, logging_obj = await self._run_pre_call_logic_with_arrival(monkeypatch, auth_completed_at=None)
|
||||
|
||||
metadata = returned_data["metadata"]
|
||||
assert "queue_time_seconds" not in metadata
|
||||
assert returned_data["proxy_server_request"]["arrival_time"] + metadata["pre_processing_seconds"] == (
|
||||
pytest.approx(logging_obj.start_time.timestamp(), abs=1e-6)
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue