scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-12 23:01:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 89

emit metric correctly

Browse source
This commit is contained in:
shivam 2026-02-21 17:00:42 -08:00
parent 26ea29afd3
commit 6f6e559502
2 changed files with 21 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
litellm/proxy/common_request_processing.py
View file

@ -563,16 +563,12 @@ class ProxyBaseLLMRequestProcessing:
llm_router: Optional[Router] = None,
) -> Tuple[dict, LiteLLMLoggingObj]:
start_time = datetime.now() # start before calling guardrail hooks
# Calculate request queue time if arrival_time is available
# Use start_time.timestamp() to avoid extra time.time() call for better performance
proxy_server_request = self.data.get("proxy_server_request", {})
arrival_time = proxy_server_request.get("arrival_time")
queue_time_seconds = None
if arrival_time is not None:
# Convert start_time (datetime) to timestamp for calculation
processing_start_time = start_time.timestamp()
queue_time_seconds = processing_start_time - arrival_time
# Capture arrival_time at the earliest moment for queue time metric.
# Must be before add_litellm_data_to_request so add_litellm preserves it.
arrival_time = start_time.timestamp()
if "proxy_server_request" not in self.data:
self.data["proxy_server_request"] = {}
self.data["proxy_server_request"]["arrival_time"] = arrival_time
self.data = await add_litellm_data_to_request(
data=self.data,
@ -583,8 +579,16 @@ class ProxyBaseLLMRequestProcessing:
proxy_config=proxy_config,
)
# Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved
if queue_time_seconds is not None:
# Calculate request queue time (time from arrival to processing start)
proxy_server_request = self.data.get("proxy_server_request", {})
arrival_time = proxy_server_request.get("arrival_time")
queue_time_seconds = None
if arrival_time is not None:
processing_start_time = start_time.timestamp()
queue_time_seconds = processing_start_time - arrival_time
# Store queue time in metadata for Prometheus litellm_request_queue_time_seconds metric
if queue_time_seconds is not None and queue_time_seconds >= 0:
from litellm.proxy.litellm_pre_call_utils import _get_metadata_variable_name
_metadata_variable_name = _get_metadata_variable_name(request)

7
litellm/proxy/litellm_pre_call_utils.py
View file

@ -847,8 +847,11 @@ async def add_litellm_data_to_request( # noqa: PLR0915
# Init - Proxy Server Request
# we do this as soon as entering so we track the original request
##########################################################
# Track arrival time for queue time metric
arrival_time = time.time()
# Track arrival time for queue time metric. Preserve from earlier in the
# request flow if set (e.g. by common_processing_pre_call_logic for accurate
# litellm_request_queue_time_seconds Prometheus metric).
existing_proxy = data.get("proxy_server_request") or {}
arrival_time = existing_proxy.get("arrival_time") or time.time()
data["proxy_server_request"] = {
"url": str(request.url),
"method": request.method,