scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

perf(prometheus): parallelize budget metrics, fix caching bug, reduce CPU by ~40% (#20544)

Browse source
This commit is contained in:
Alexsander Hamir 2026-02-06 09:18:24 -08:00 • committed by Ishaan Jaffer
parent c0fb179a32
commit 461fff1a87
1 changed files with 31 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
litellm/integrations/prometheus.py
View file

@ -1,6 +1,7 @@
# used for /metrics endpoint on LiteLLM Proxy
#### What this does ####
# On success, log events to Prometheus
import asyncio
import os
import sys
from datetime import datetime, timedelta
@ -1198,28 +1199,34 @@ class PrometheusLogger(CustomLogger):
"user_api_key_user_max_budget", None
)
await self._set_api_key_budget_metrics_after_api_request(
user_api_key=user_api_key,
user_api_key_alias=user_api_key_alias,
response_cost=response_cost,
key_max_budget=_api_key_max_budget,
key_spend=_api_key_spend,
)
await self._set_team_budget_metrics_after_api_request(
user_api_team=user_api_team,
user_api_team_alias=user_api_team_alias,
team_spend=_team_spend,
team_max_budget=_team_max_budget,
response_cost=response_cost,
)
await self._set_user_budget_metrics_after_api_request(
user_id=user_id,
user_spend=_user_spend,
user_max_budget=_user_max_budget,
response_cost=response_cost,
results = await asyncio.gather(
self._set_api_key_budget_metrics_after_api_request(
user_api_key=user_api_key,
user_api_key_alias=user_api_key_alias,
response_cost=response_cost,
key_max_budget=_api_key_max_budget,
key_spend=_api_key_spend,
),
self._set_team_budget_metrics_after_api_request(
user_api_team=user_api_team,
user_api_team_alias=user_api_team_alias,
team_spend=_team_spend,
team_max_budget=_team_max_budget,
response_cost=response_cost,
),
self._set_user_budget_metrics_after_api_request(
user_id=user_id,
user_spend=_user_spend,
user_max_budget=_user_max_budget,
response_cost=response_cost,
),
return_exceptions=True,
)
for i, r in enumerate(results):
if isinstance(r, Exception):
verbose_logger.debug(
f"[Non-Blocking] Prometheus: Budget metric lookup {['key', 'team', 'user'][i]} failed: {r}"
)
def _increment_top_level_request_and_spend_metrics(
self,
@ -2738,12 +2745,14 @@ class PrometheusLogger(CustomLogger):
max_budget=max_budget,
)
try:
# Note: Setting check_db_only=True bypasses cache and hits DB on every request,
# causing huge latency increase and CPU spikes. Keep check_db_only=False.
user_info = await get_user_object(
user_id=user_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
user_id_upsert=False,
check_db_only=True,
check_db_only=False,
)
except Exception as e:
verbose_logger.debug(