scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

fix(lowest_latency.py): revert changes -> fix cpu utilization issue

Browse source
This commit is contained in:
Krrish Dholakia 2024-05-02 22:05:56 -07:00
parent 91971fa9e0
commit 044cebd68f
1 changed files with 7 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
litellm/router_strategy/lowest_latency.py
View file

@ -4,7 +4,6 @@ from pydantic import BaseModel, Extra, Field, root_validator
import dotenv, os, requests, random import dotenv, os, requests, random
from typing import Optional, Union, List, Dict from typing import Optional, Union, List, Dict
from datetime import datetime, timedelta from datetime import datetime, timedelta
import random
dotenv.load_dotenv() # Loading env variables using dotenv dotenv.load_dotenv() # Loading env variables using dotenv
import traceback import traceback
@ -30,7 +29,6 @@ class LiteLLMBase(BaseModel):
class RoutingArgs(LiteLLMBase): class RoutingArgs(LiteLLMBase):
ttl: int = 1 * 60 * 60 # 1 hour ttl: int = 1 * 60 * 60 # 1 hour
lowest_latency_buffer: float = 0
class LowestLatencyLoggingHandler(CustomLogger): class LowestLatencyLoggingHandler(CustomLogger):
@ -316,12 +314,8 @@ class LowestLatencyLoggingHandler(CustomLogger):
# randomly sample from all_deployments, incase all deployments have latency=0.0 # randomly sample from all_deployments, incase all deployments have latency=0.0
_items = all_deployments.items() _items = all_deployments.items()
all_deployments = random.sample(list(_items), len(_items)) all_deployments = random.sample(list(_items), len(_items))
all_deployments = dict(all_deployments) all_deployments = dict(all_deployments)
### GET AVAILABLE DEPLOYMENTS ### filter out any deployments > tpm/rpm limits
potential_deployments = []
for item, item_map in all_deployments.items(): for item, item_map in all_deployments.items():
## get the item from model list ## get the item from model list
_deployment = None _deployment = None
@ -370,33 +364,17 @@ class LowestLatencyLoggingHandler(CustomLogger):
# End of Debugging Logic # End of Debugging Logic
# -------------- # # -------------- #
if ( if item_latency == 0:
deployment = _deployment
break
elif (
item_tpm + input_tokens > _deployment_tpm item_tpm + input_tokens > _deployment_tpm
or item_rpm + 1 > _deployment_rpm or item_rpm + 1 > _deployment_rpm
): # if user passed in tpm / rpm in the model_list ): # if user passed in tpm / rpm in the model_list
continue continue
else: elif item_latency < lowest_latency:
potential_deployments.append((_deployment, item_latency)) lowest_latency = item_latency
deployment = _deployment
if len(potential_deployments) == 0:
return None
# Sort potential deployments by latency
sorted_deployments = sorted(potential_deployments, key=lambda x: x[1])
# Find lowest latency deployment
lowest_latency = sorted_deployments[0][1]
# Find deployments within buffer of lowest latency
buffer = self.routing_args.lowest_latency_buffer * lowest_latency
valid_deployments = [
x for x in sorted_deployments if x[1] <= lowest_latency + buffer
]
# Pick a random deployment from valid deployments
random_valid_deployment = random.choice(valid_deployments)
deployment = random_valid_deployment[0]
if request_kwargs is not None and "metadata" in request_kwargs: if request_kwargs is not None and "metadata" in request_kwargs:
request_kwargs["metadata"][ request_kwargs["metadata"][
"_latency_per_deployment" "_latency_per_deployment"