mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
perf(router): optimize _filter_cooldown_deployments to O(n) (#15091)
Refactored to use set-based lookup and list comprehension instead of two-pass approach with list.remove(). Old complexity: O(n×m + k×n) - First loop: n deployments × m list lookups = O(n×m) - Second loop: k removals × n list.remove() scans = O(k×n) New complexity: O(m + n) - Convert to set: O(m) - Filter with O(1) set lookups: O(n) Example with 100 deployments, 5 in cooldown: - Old: ~1000 operations - New: ~105 operations Called on every request - high impact for production.
This commit is contained in:
parent
0ca11eefde
commit
26145da3e7
1 changed files with 6 additions and 12 deletions
|
|
@ -7259,19 +7259,13 @@ class Router:
|
|||
Returns:
|
||||
List of healthy deployments
|
||||
"""
|
||||
# filter out the deployments currently cooling down
|
||||
deployments_to_remove = []
|
||||
verbose_router_logger.debug(f"cooldown deployments: {cooldown_deployments}")
|
||||
# Find deployments in model_list whose model_id is cooling down
|
||||
for deployment in healthy_deployments:
|
||||
deployment_id = deployment["model_info"]["id"]
|
||||
if deployment_id in cooldown_deployments:
|
||||
deployments_to_remove.append(deployment)
|
||||
|
||||
# remove unhealthy deployments from healthy deployments
|
||||
for deployment in deployments_to_remove:
|
||||
healthy_deployments.remove(deployment)
|
||||
return healthy_deployments
|
||||
# Convert to set for O(1) lookup and use list comprehension for O(n) filtering
|
||||
cooldown_set = set(cooldown_deployments)
|
||||
return [
|
||||
deployment for deployment in healthy_deployments
|
||||
if deployment["model_info"]["id"] not in cooldown_set
|
||||
]
|
||||
|
||||
def _track_deployment_metrics(
|
||||
self, deployment, parent_otel_span: Optional[Span], response=None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue