From 26145da3e744b072ad7069cf4a99e2a9b717ce9e Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Tue, 30 Sep 2025 18:39:12 -0700 Subject: [PATCH] perf(router): optimize _filter_cooldown_deployments to O(n) (#15091) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refactored to use set-based lookup and list comprehension instead of two-pass approach with list.remove(). Old complexity: O(n×m + k×n) - First loop: n deployments × m list lookups = O(n×m) - Second loop: k removals × n list.remove() scans = O(k×n) New complexity: O(m + n) - Convert to set: O(m) - Filter with O(1) set lookups: O(n) Example with 100 deployments, 5 in cooldown: - Old: ~1000 operations - New: ~105 operations Called on every request - high impact for production. --- litellm/router.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 59c3235c039..9dcdca288fa 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7259,19 +7259,13 @@ class Router: Returns: List of healthy deployments """ - # filter out the deployments currently cooling down - deployments_to_remove = [] verbose_router_logger.debug(f"cooldown deployments: {cooldown_deployments}") - # Find deployments in model_list whose model_id is cooling down - for deployment in healthy_deployments: - deployment_id = deployment["model_info"]["id"] - if deployment_id in cooldown_deployments: - deployments_to_remove.append(deployment) - - # remove unhealthy deployments from healthy deployments - for deployment in deployments_to_remove: - healthy_deployments.remove(deployment) - return healthy_deployments + # Convert to set for O(1) lookup and use list comprehension for O(n) filtering + cooldown_set = set(cooldown_deployments) + return [ + deployment for deployment in healthy_deployments + if deployment["model_info"]["id"] not in cooldown_set + ] def _track_deployment_metrics( self, deployment, parent_otel_span: Optional[Span], response=None