From 048b4cd6eab35a219f3c30dfa0acfd63495a9cbb Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Thu, 26 Sep 2024 19:35:59 -0700
Subject: [PATCH] fix parallel request limiter - use one cache update call

---
 litellm/caching.py                            |  2 +-
 .../proxy/hooks/parallel_request_limiter.py   | 41 +++++--------------
 litellm/proxy/proxy_config.yaml               |  2 +-
 3 files changed, 13 insertions(+), 32 deletions(-)

diff --git a/litellm/caching.py b/litellm/caching.py
index 9bb03b99af8..05e330c0699 100644
--- a/litellm/caching.py
+++ b/litellm/caching.py
@@ -2038,7 +2038,7 @@ class DualCache(BaseCache):
 
             if self.redis_cache is not None and local_only == False:
                 await self.redis_cache.async_set_cache_pipeline(
-                    cache_list=cache_list, ttl=kwargs.get("ttl", None), **kwargs
+                    cache_list=cache_list, ttl=kwargs.pop("ttl", None), **kwargs
                 )
         except Exception as e:
             verbose_logger.exception(
diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py
index 7eaf515f2f4..04aa5ec843a 100644
--- a/litellm/proxy/hooks/parallel_request_limiter.py
+++ b/litellm/proxy/hooks/parallel_request_limiter.py
@@ -472,6 +472,8 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
             # Update usage - API Key
             # ------------
 
+            values_to_update_in_cache = []
+
             if user_api_key is not None:
                 request_count_api_key = (
                     f"{user_api_key}::{precise_minute}::request_count"
@@ -495,12 +497,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
                 self.print_verbose(
                     f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
                 )
-                await self.internal_usage_cache.async_set_cache(
-                    request_count_api_key,
-                    new_val,
-                    ttl=60,
-                    litellm_parent_otel_span=litellm_parent_otel_span,
-                )  # store in cache for 1 min.
+                values_to_update_in_cache.append((request_count_api_key, new_val))
 
             # ------------
             # Update usage - model group + API Key
@@ -536,12 +533,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
                 self.print_verbose(
                     f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
                 )
-                await self.internal_usage_cache.async_set_cache(
-                    request_count_api_key,
-                    new_val,
-                    ttl=60,
-                    litellm_parent_otel_span=litellm_parent_otel_span,
-                )
+                values_to_update_in_cache.append((request_count_api_key, new_val))
 
             # ------------
             # Update usage - User
@@ -574,12 +566,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
                 self.print_verbose(
                     f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
                 )
-                await self.internal_usage_cache.async_set_cache(
-                    request_count_api_key,
-                    new_val,
-                    ttl=60,
-                    litellm_parent_otel_span=litellm_parent_otel_span,
-                )  # store in cache for 1 min.
+                values_to_update_in_cache.append((request_count_api_key, new_val))
 
             # ------------
             # Update usage - Team
@@ -612,12 +599,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
                 self.print_verbose(
                     f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
                 )
-                await self.internal_usage_cache.async_set_cache(
-                    request_count_api_key,
-                    new_val,
-                    ttl=60,
-                    litellm_parent_otel_span=litellm_parent_otel_span,
-                )  # store in cache for 1 min.
+                values_to_update_in_cache.append((request_count_api_key, new_val))
 
             # ------------
             # Update usage - End User
@@ -650,13 +632,12 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
                 self.print_verbose(
                     f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
                 )
-                await self.internal_usage_cache.async_set_cache(
-                    request_count_api_key,
-                    new_val,
-                    ttl=60,
-                    litellm_parent_otel_span=litellm_parent_otel_span,
-                )  # store in cache for 1 min.
+                values_to_update_in_cache.append((request_count_api_key, new_val))
 
+            await self.internal_usage_cache.dual_cache.async_batch_set_cache(
+                cache_list=values_to_update_in_cache,
+                ttl=60,
+            )
         except Exception as e:
             self.print_verbose(e)  # noqa
 
diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml
index dda6f82749c..c8ca606a9ec 100644
--- a/litellm/proxy/proxy_config.yaml
+++ b/litellm/proxy/proxy_config.yaml
@@ -1,5 +1,5 @@
 model_list:
- - model_name: gpt-3.5-turbo
+ - model_name: db-openai-endpoint
    litellm_params:
      model: openai/gpt-3.5-turbo
      api_key: fake-key