From 2dd5f2bc8c41e9060f1885b9640fd0553276efa5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Mar 2024 20:01:36 -0700 Subject: [PATCH] fix(tpm_rpm_limiter.py): enable redis caching for tpm/rpm checks on keys/user/teams allows tpm/rpm checks to work across instances https://github.com/BerriAI/litellm/issues/2730 --- litellm/proxy/utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 6e1f0521f26..7d3e30f865c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -48,14 +48,12 @@ class ProxyLogging: """ def __init__( - self, - user_api_key_cache: DualCache, - redis_usage_cache: Optional[RedisCache] = None, + self, user_api_key_cache: DualCache, redis_usage_cache: Optional[RedisCache] ): ## INITIALIZE LITELLM CALLBACKS ## self.call_details: dict = {} self.call_details["user_api_key_cache"] = user_api_key_cache - self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler() + # self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler() self.max_tpm_rpm_limiter = _PROXY_MaxTPMRPMLimiter( redis_usage_cache=redis_usage_cache ) @@ -74,7 +72,7 @@ class ProxyLogging: def _init_litellm_callbacks(self): print_verbose(f"INITIALIZING LITELLM CALLBACKS!") - litellm.callbacks.append(self.max_parallel_request_limiter) + # litellm.callbacks.append(self.max_parallel_request_limiter) litellm.callbacks.append(self.max_tpm_rpm_limiter) litellm.callbacks.append(self.max_budget_limiter) litellm.callbacks.append(self.cache_control_check)