fix failed unit-tests

litellm/router.py

@@ -10469,11 +10469,23 @@ class Router:
                     llm_provider="",
                 )
 
-            # 5. Apply load balancing strategy
+            # 5. Determine and apply routing strategy based on priority hierarchy
+            winning_strategy = self._get_routing_strategy_for_request(model, request_kwargs)
+            strategy, selector = self._get_routing_context(model, request_kwargs, winning_strategy)
+
+            # 6. Apply load balancing strategy
             start_time = time.perf_counter()
 
-            # Get the selector for the winning strategy (already determined above)
-            strategy, selector = self._get_routing_context(model, request_kwargs, winning_strategy)
+            # Select deployment using the winning strategy
+            deployment = await self._select_deployment_async(
+                strategy=strategy,
+                selector=selector,
+                model=model,
+                healthy_deployments=pass_through_deployments,
+                messages=messages,
+                input=input,
+                request_kwargs=request_kwargs,
+            )
 
             # Default to simple-shuffle if strategy didn't return anything
             if deployment is None:
@@ -10610,6 +10622,14 @@ class Router:
         # Ensure the logger for the winning strategy is initialized
         self._ensure_routing_strategy_logger(winning_strategy, self.routing_strategy_args)
 
+        # 1. Perform common checks to get healthy deployments list
+        model, healthy_deployments = self._common_checks_available_deployment(
+            model=model,
+            messages=messages,
+            input=input,
+            specific_deployment=specific_deployment,
+            request_kwargs=request_kwargs,
+        )
 
         if isinstance(healthy_deployments, dict):
             return healthy_deployments
@@ -10691,16 +10711,8 @@ class Router:
             )
             # if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm
             ############## Check 'weight' param set for weighted pick #################
-            return simple_shuffle(
-                llm_router_instance=self,
-                healthy_deployments=healthy_deployments,
-                model=model,
-            )
-        return simple_shuffle(
-            llm_router_instance=self,
-            healthy_deployments=healthy_deployments,
-            model=model,
-        )
+            return deployment
+        return deployment
 
     def get_available_deployment_for_pass_through(
         self,

litellm/router_strategy/lowest_cost.py

@@ -190,145 +190,6 @@ class LowestCostLoggingHandler(CustomLogger):
             )
             pass
 
-    def get_available_deployments(  # noqa: PLR0915
-        self,
-        model_group: str,
-        healthy_deployments: list,
-        messages: Optional[List[Dict[str, str]]] = None,
-        input: Optional[Union[str, List]] = None,
-        request_kwargs: Optional[Dict] = None,
-    ):
-        """
-        Returns a deployment with the lowest cost
-        """
-        cost_key = f"{model_group}_map"
-
-        request_count_dict = self.router_cache.get_cache(key=cost_key) or {}
-
-        # -----------------------
-        # Find lowest used model
-        # ----------------------
-        float("inf")
-
-        current_date = datetime.now().strftime("%Y-%m-%d")
-        current_hour = datetime.now().strftime("%H")
-        current_minute = datetime.now().strftime("%M")
-        precise_minute = f"{current_date}-{current_hour}-{current_minute}"
-
-        if request_count_dict is None:  # base case
-            return
-
-        all_deployments = request_count_dict
-        for d in healthy_deployments:
-            ## if healthy deployment not yet used
-            if d["model_info"]["id"] not in all_deployments:
-                all_deployments[d["model_info"]["id"]] = {
-                    precise_minute: {"tpm": 0, "rpm": 0},
-                }
-
-        try:
-            input_tokens = token_counter(messages=messages, text=input)
-        except Exception:
-            input_tokens = 0
-
-        # randomly sample from all_deployments, incase all deployments have latency=0.0
-        _items = all_deployments.items()
-
-        ### GET AVAILABLE DEPLOYMENTS ### filter out any deployments > tpm/rpm limits
-        potential_deployments = []
-        _cost_per_deployment = {}
-        for item, item_map in all_deployments.items():
-            ## get the item from model list
-            _deployment = None
-            for m in healthy_deployments:
-                if item == m["model_info"]["id"]:
-                    _deployment = m
-
-            if _deployment is None:
-                continue  # skip to next one
-
-            _deployment_tpm = (
-                _deployment.get("tpm", None)
-                or _deployment.get("litellm_params", {}).get("tpm", None)
-                or _deployment.get("model_info", {}).get("tpm", None)
-                or float("inf")
-            )
-
-            _deployment_rpm = (
-                _deployment.get("rpm", None)
-                or _deployment.get("litellm_params", {}).get("rpm", None)
-                or _deployment.get("model_info", {}).get("rpm", None)
-                or float("inf")
-            )
-            item_litellm_model_name = _deployment.get("litellm_params", {}).get("model")
-            item_litellm_model_cost_map = litellm.model_cost.get(
-                item_litellm_model_name, {}
-            )
-
-            # check if user provided input_cost_per_token and output_cost_per_token in litellm_params
-            item_input_cost = None
-            item_output_cost = None
-            if _deployment.get("litellm_params", {}).get("input_cost_per_token", None):
-                item_input_cost = _deployment.get("litellm_params", {}).get(
-                    "input_cost_per_token"
-                )
-
-            if _deployment.get("litellm_params", {}).get("output_cost_per_token", None):
-                item_output_cost = _deployment.get("litellm_params", {}).get(
-                    "output_cost_per_token"
-                )
-
-            if item_input_cost is None:
-                item_input_cost = item_litellm_model_cost_map.get(
-                    "input_cost_per_token", 5.0
-                )
-
-            if item_output_cost is None:
-                item_output_cost = item_litellm_model_cost_map.get(
-                    "output_cost_per_token", 5.0
-                )
-
-            # if litellm["model"] is not in model_cost map -> use item_cost = $10
-
-            item_cost = item_input_cost + item_output_cost
-
-            item_rpm = item_map.get(precise_minute, {}).get("rpm", 0)
-            item_tpm = item_map.get(precise_minute, {}).get("tpm", 0)
-
-            verbose_router_logger.debug(
-                f"item_cost: {item_cost}, item_tpm: {item_tpm}, item_rpm: {item_rpm}, model_id: {_deployment.get('model_info', {}).get('id')}"
-            )
-
-            # -------------- #
-            # Debugging Logic
-            # -------------- #
-            # We use _cost_per_deployment to log to langfuse, slack - this is not used to make a decision on routing
-            # this helps a user to debug why the router picked a specfic deployment      #
-            _deployment_api_base = _deployment.get("litellm_params", {}).get(
-                "api_base", ""
-            )
-            if _deployment_api_base is not None:
-                _cost_per_deployment[_deployment_api_base] = item_cost
-            # -------------- #
-            # End of Debugging Logic
-            # -------------- #
-
-            if (
-                item_tpm + input_tokens > _deployment_tpm
-                or item_rpm + 1 > _deployment_rpm
-            ):  # if user passed in tpm / rpm in the model_list
-                continue
-            else:
-                potential_deployments.append((_deployment, item_cost))
-
-        if len(potential_deployments) == 0:
-            return None
-
-        potential_deployments = sorted(potential_deployments, key=lambda x: x[1])
-
-        selected_deployment = potential_deployments[0][0]
-        return selected_deployment
-
     async def async_get_available_deployments(  # noqa: PLR0915
         self,
         model_group: str,