scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 79

fix(tpm_rpm_limiter.py): enable redis caching for tpm/rpm checks on keys/user/teams

Browse source 
allows tpm/rpm checks to work across instances

https://github.com/BerriAI/litellm/issues/2730
This commit is contained in:
Krrish Dholakia 2024-03-30 20:01:36 -07:00 • committed by Ishaan Jaff
parent ddb35facc0
commit 2dd5f2bc8c
1 changed files with 3 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/proxy/utils.py
View file

@ -48,14 +48,12 @@ class ProxyLogging:
"""
def __init__(
self,
user_api_key_cache: DualCache,
redis_usage_cache: Optional[RedisCache] = None,
self, user_api_key_cache: DualCache, redis_usage_cache: Optional[RedisCache]
):
## INITIALIZE LITELLM CALLBACKS ##
self.call_details: dict = {}
self.call_details["user_api_key_cache"] = user_api_key_cache
self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler()
# self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler()
self.max_tpm_rpm_limiter = _PROXY_MaxTPMRPMLimiter(
redis_usage_cache=redis_usage_cache
)
@ -74,7 +72,7 @@ class ProxyLogging:
def _init_litellm_callbacks(self):
print_verbose(f"INITIALIZING LITELLM CALLBACKS!")
litellm.callbacks.append(self.max_parallel_request_limiter)
# litellm.callbacks.append(self.max_parallel_request_limiter)
litellm.callbacks.append(self.max_tpm_rpm_limiter)
litellm.callbacks.append(self.max_budget_limiter)
litellm.callbacks.append(self.cache_control_check)