From 984259d4204ed1e41cb9d1c4a0d35a5984427cc2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 11:22:52 -0700 Subject: [PATCH 1/5] temp - show better debug logs for lowest latency --- litellm/integrations/slack_alerting.py | 6 ++++++ litellm/router_strategy/lowest_latency.py | 13 +++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 64f9b538456..46eba3a23a3 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -140,6 +140,7 @@ class SlackAlerting: raise e def _get_deployment_latencies_to_alert(self, metadata=None): + print("inside get deployment latencies metadata", metadata) # noqa if metadata is None: return None @@ -188,6 +189,7 @@ class SlackAlerting: request_info=request_info, kwargs=kwargs ) # add deployment latencies to alert + print("in response taking too long callback, kwargs: ", kwargs) # noqa if ( kwargs is not None and "litellm_params" in kwargs @@ -281,6 +283,10 @@ class SlackAlerting: f"`Requests are hanging - {self.alerting_threshold}s+ request time`" ) + print( + "inside hanging request callback, request_data: ", request_data + ) # noqa + # add deployment latencies to alert _deployment_latency_map = self._get_deployment_latencies_to_alert( metadata=request_data.get("metadata", {}) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 221a666dcac..998336fa12b 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -339,12 +339,19 @@ class LowestLatencyLoggingHandler(CustomLogger): item_rpm = item_map.get(precise_minute, {}).get("rpm", 0) item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) + # _latency_per_deployment is used for debuggig + _deployment_api_base = _deployment.get("litellm_params", {}).get( + "api_base", "" + ) + # get average latency total: float = 0.0 for _call_latency in item_latency: if isinstance(_call_latency, float): total += _call_latency item_latency = total / len(item_latency) + print("item_latency=", item_latency, "deployment=", deployment) # noqa + _latency_per_deployment[_deployment_api_base] = item_latency if item_latency == 0: deployment = _deployment break @@ -356,12 +363,6 @@ class LowestLatencyLoggingHandler(CustomLogger): elif item_latency < lowest_latency: lowest_latency = item_latency deployment = _deployment - - # _latency_per_deployment is used for debuggig - _deployment_api_base = _deployment.get("litellm_params", {}).get( - "api_base", "" - ) - _latency_per_deployment[_deployment_api_base] = item_latency if request_kwargs is not None and "metadata" in request_kwargs: request_kwargs["metadata"][ "_latency_per_deployment" From 787735bb5a2bb256df103c8314183ff9c6d9eddc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 11:25:03 -0700 Subject: [PATCH 2/5] fix --- litellm/router_strategy/lowest_latency.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 998336fa12b..f0fc63267fb 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -351,7 +351,8 @@ class LowestLatencyLoggingHandler(CustomLogger): total += _call_latency item_latency = total / len(item_latency) print("item_latency=", item_latency, "deployment=", deployment) # noqa - _latency_per_deployment[_deployment_api_base] = item_latency + if _deployment_api_base is not None: + _latency_per_deployment[_deployment_api_base] = item_latency if item_latency == 0: deployment = _deployment break From 737af2b45869f8a0264067235d504d65230bfce7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 11:35:08 -0700 Subject: [PATCH 3/5] fix better debugging for latency --- litellm/router_strategy/lowest_latency.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index f0fc63267fb..6c7454f74a1 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -339,18 +339,20 @@ class LowestLatencyLoggingHandler(CustomLogger): item_rpm = item_map.get(precise_minute, {}).get("rpm", 0) item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) - # _latency_per_deployment is used for debuggig - _deployment_api_base = _deployment.get("litellm_params", {}).get( - "api_base", "" - ) - # get average latency total: float = 0.0 for _call_latency in item_latency: if isinstance(_call_latency, float): total += _call_latency item_latency = total / len(item_latency) - print("item_latency=", item_latency, "deployment=", deployment) # noqa + print("item_latency=", item_latency, "deployment=", _deployment) # noqa + + # Debugging Logic # + _deployment_api_base = _deployment.get("litellm_params", {}).get( + "api_base", "" + ) + # End of Debugging Logic # + if _deployment_api_base is not None: _latency_per_deployment[_deployment_api_base] = item_latency if item_latency == 0: From bf92a0b31c0b2c01993f108feb266772bf8f24dd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 19:34:28 -0700 Subject: [PATCH 4/5] fix debugging lowest latency router --- litellm/router_strategy/lowest_latency.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 6c7454f74a1..19780f708d7 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -345,16 +345,21 @@ class LowestLatencyLoggingHandler(CustomLogger): if isinstance(_call_latency, float): total += _call_latency item_latency = total / len(item_latency) - print("item_latency=", item_latency, "deployment=", _deployment) # noqa - # Debugging Logic # + # -------------- # + # Debugging Logic + # -------------- # + # We use _latency_per_deployment to log to langfuse, slack - this is not used to make a decision on routing + # this helps a user to debug why the router picked a specfic deployment # _deployment_api_base = _deployment.get("litellm_params", {}).get( "api_base", "" ) - # End of Debugging Logic # - if _deployment_api_base is not None: _latency_per_deployment[_deployment_api_base] = item_latency + # -------------- # + # End of Debugging Logic + # -------------- # + if item_latency == 0: deployment = _deployment break From 24e918b10f2924dce6315121bbcd01b5f77da30a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Apr 2024 19:36:42 -0700 Subject: [PATCH 5/5] fix clean up prints from slack alerting --- litellm/integrations/slack_alerting.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 46eba3a23a3..9151aecb010 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -140,8 +140,6 @@ class SlackAlerting: raise e def _get_deployment_latencies_to_alert(self, metadata=None): - print("inside get deployment latencies metadata", metadata) # noqa - if metadata is None: return None @@ -189,7 +187,6 @@ class SlackAlerting: request_info=request_info, kwargs=kwargs ) # add deployment latencies to alert - print("in response taking too long callback, kwargs: ", kwargs) # noqa if ( kwargs is not None and "litellm_params" in kwargs @@ -283,10 +280,6 @@ class SlackAlerting: f"`Requests are hanging - {self.alerting_threshold}s+ request time`" ) - print( - "inside hanging request callback, request_data: ", request_data - ) # noqa - # add deployment latencies to alert _deployment_latency_map = self._get_deployment_latencies_to_alert( metadata=request_data.get("metadata", {}) @@ -438,12 +431,6 @@ class SlackAlerting: level: str - Low|Medium|High - if calls might fail (Medium) or are failing (High); Currently, no alerts would be 'Low'. message: str - what is the alert about """ - print( - "inside send alert for slack, message: ", - message, - "self.alerting: ", - self.alerting, - ) if self.alerting is None: return