scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-11 03:38:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

fix parallel request limiter - use one cache update call

Browse source
This commit is contained in:
Ishaan Jaff 2024-09-26 19:35:59 -07:00
parent 480a8d73c0
commit 048b4cd6ea
3 changed files with 13 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/caching.py
View file

@ -2038,7 +2038,7 @@ class DualCache(BaseCache):
if self.redis_cache is not None and local_only == False:
await self.redis_cache.async_set_cache_pipeline(
cache_list=cache_list, ttl=kwargs.get("ttl", None), **kwargs
cache_list=cache_list, ttl=kwargs.pop("ttl", None), **kwargs
)
except Exception as e:
verbose_logger.exception(

41
litellm/proxy/hooks/parallel_request_limiter.py
View file

@ -472,6 +472,8 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - API Key
# ------------
values_to_update_in_cache = []
if user_api_key is not None:
request_count_api_key = (
f"{user_api_key}::{precise_minute}::request_count"
@ -495,12 +497,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
self.print_verbose(
f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
)
await self.internal_usage_cache.async_set_cache(
request_count_api_key,
new_val,
ttl=60,
litellm_parent_otel_span=litellm_parent_otel_span,
) # store in cache for 1 min.
values_to_update_in_cache.append((request_count_api_key, new_val))
# ------------
# Update usage - model group + API Key
@ -536,12 +533,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
self.print_verbose(
f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
)
await self.internal_usage_cache.async_set_cache(
request_count_api_key,
new_val,
ttl=60,
litellm_parent_otel_span=litellm_parent_otel_span,
)
values_to_update_in_cache.append((request_count_api_key, new_val))
# ------------
# Update usage - User
@ -574,12 +566,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
self.print_verbose(
f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
)
await self.internal_usage_cache.async_set_cache(
request_count_api_key,
new_val,
ttl=60,
litellm_parent_otel_span=litellm_parent_otel_span,
) # store in cache for 1 min.
values_to_update_in_cache.append((request_count_api_key, new_val))
# ------------
# Update usage - Team
@ -612,12 +599,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
self.print_verbose(
f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
)
await self.internal_usage_cache.async_set_cache(
request_count_api_key,
new_val,
ttl=60,
litellm_parent_otel_span=litellm_parent_otel_span,
) # store in cache for 1 min.
values_to_update_in_cache.append((request_count_api_key, new_val))
# ------------
# Update usage - End User
@ -650,13 +632,12 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
self.print_verbose(
f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
)
await self.internal_usage_cache.async_set_cache(
request_count_api_key,
new_val,
ttl=60,
litellm_parent_otel_span=litellm_parent_otel_span,
) # store in cache for 1 min.
values_to_update_in_cache.append((request_count_api_key, new_val))
await self.internal_usage_cache.dual_cache.async_batch_set_cache(
cache_list=values_to_update_in_cache,
ttl=60,
)
except Exception as e:
self.print_verbose(e) # noqa

2
litellm/proxy/proxy_config.yaml
View file

@ -1,5 +1,5 @@
model_list:
- model_name: gpt-3.5-turbo
- model_name: db-openai-endpoint
litellm_params:
model: openai/gpt-3.5-turbo
api_key: fake-key