scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-24 00:52:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 137

fix lowest latency tests

Browse source
This commit is contained in:
CLARKBENHAM 2024-04-02 19:10:40 -07:00
parent 29573b0967
commit 164898a213
2 changed files with 16 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
litellm/router_strategy/lowest_latency.py
View file

@ -35,9 +35,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
logged_success: int = 0
logged_failure: int = 0
def __init__(
self, router_cache: DualCache, model_list: list, routing_args: dict = {}
):
def __init__(self, router_cache: DualCache, model_list: list, routing_args: dict = {}):
self.router_cache = router_cache
self.model_list = model_list
self.routing_args = RoutingArgs(**routing_args)
@ -50,9 +48,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
if kwargs["litellm_params"].get("metadata") is None:
pass
else:
model_group = kwargs["litellm_params"]["metadata"].get(
"model_group", None
)
model_group = kwargs["litellm_params"]["metadata"].get("model_group", None)
id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
if model_group is None or id is None:
@ -65,8 +61,8 @@ class LowestLatencyLoggingHandler(CustomLogger):
{
{model_group}_map: {
id: {
"latency": [..]
f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}
"latency": [..]
f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}
}
}
}
@ -132,9 +128,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
if kwargs["litellm_params"].get("metadata") is None:
pass
else:
model_group = kwargs["litellm_params"]["metadata"].get(
"model_group", None
)
model_group = kwargs["litellm_params"]["metadata"].get("model_group", None)
id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
if model_group is None or id is None:
@ -147,8 +141,8 @@ class LowestLatencyLoggingHandler(CustomLogger):
{
{model_group}_map: {
id: {
"latency": [..]
f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}
"latency": [..]
f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}
}
}
}
@ -287,13 +281,10 @@ class LowestLatencyLoggingHandler(CustomLogger):
deployment = _deployment
break
elif (
item_tpm + input_tokens > _deployment_tpm
or item_rpm + 1 > _deployment_rpm
item_tpm + input_tokens > _deployment_tpm or item_rpm + 1 > _deployment_rpm
): # if user passed in tpm / rpm in the model_list
continue
elif item_latency < lowest_latency:
lowest_latency = item_latency
deployment = _deployment
if deployment is None:
deployment = random.choice(healthy_deployments)
return deployment

30
litellm/tests/test_lowest_latency_routing.py
View file

@ -230,19 +230,11 @@ def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm):
d2 = [(lowest_latency_logger, "5678", 50, 0.01)] * non_ans_rpm
asyncio.run(_gather_deploy([*d1, *d2]))
## CHECK WHAT'S SELECTED ##
print(dir(lowest_latency_logger))
print(
"availible",
lowest_latency_logger.get_available_deployments(
model_group=model_group, healthy_deployments=model_list
),
)
assert (
lowest_latency_logger.get_available_deployments(
model_group=model_group, healthy_deployments=model_list
)["model_info"]["id"]
== ans
d_ans = lowest_latency_logger.get_available_deployments(
model_group=model_group, healthy_deployments=model_list
)
print(d_ans)
assert (d_ans and d_ans["model_info"]["id"]) == ans
# test_get_available_endpoints_tpm_rpm_check_async()
@ -325,17 +317,11 @@ def test_get_available_endpoints_tpm_rpm_check(ans_rpm):
)
## CHECK WHAT'S SELECTED ##
print(
lowest_latency_logger.get_available_deployments(
model_group=model_group, healthy_deployments=model_list
)
)
assert (
lowest_latency_logger.get_available_deployments(
model_group=model_group, healthy_deployments=model_list
)["model_info"]["id"]
== ans
d_ans = lowest_latency_logger.get_available_deployments(
model_group=model_group, healthy_deployments=model_list
)
print(d_ans)
assert (d_ans and d_ans["model_info"]["id"]) == ans
def test_router_get_available_deployments():