From 2267aeb803185807212173bcfbffeea894951173 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Fri, 12 Apr 2024 18:03:23 -0700
Subject: [PATCH] fix(router.py): create a semaphore for each deployment with
 rpm

run semaphore logic for each deployment with rpm
---
 litellm/router.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/litellm/router.py b/litellm/router.py
index 6763caa39f4..c127c3f8b62 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -487,7 +487,9 @@ class Router:
                 }
             )
 
-            rpm_semaphore = self.semaphore
+            rpm_semaphore = self._get_client(
+                deployment=deployment, kwargs=kwargs, client_type="rpm_client"
+            )
 
             if rpm_semaphore is not None and isinstance(
                 rpm_semaphore, asyncio.Semaphore
@@ -1723,16 +1725,15 @@ class Router:
         model_name = litellm_params.get("model")
         model_id = model["model_info"]["id"]
         # ### IF RPM SET - initialize a semaphore ###
-        # rpm = litellm_params.get("rpm", None)
-        # print(f"rpm: {rpm}")
-        # if rpm:
-        #     semaphore = asyncio.Semaphore(rpm)
-        #     cache_key = f"{model_id}_rpm_client"
-        #     self.cache.set_cache(
-        #         key=cache_key,
-        #         value=semaphore,
-        #         local_only=True,
-        #     )
+        rpm = litellm_params.get("rpm", None)
+        if rpm:
+            semaphore = asyncio.Semaphore(rpm)
+            cache_key = f"{model_id}_rpm_client"
+            self.cache.set_cache(
+                key=cache_key,
+                value=semaphore,
+                local_only=True,
+            )
 
         #     print("STORES SEMAPHORE IN CACHE")