From 3aba9019ac27481361c3fa87956907f5c41dd4ca Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 2 Mar 2024 10:06:33 -0800 Subject: [PATCH 1/4] (feat) track used api_base in response --- litellm/router.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 6f33d0b0d5c..59cbf43d56c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -357,6 +357,11 @@ class Router: verbose_router_logger.info( f"litellm.completion(model={model_name})\033[32m 200 OK\033[0m" ) + # set used api_base in response + if hasattr(response, "_hidden_params"): + response._hidden_params["api_base"] = deployment.get( + "litellm_params", {} + ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( @@ -446,6 +451,12 @@ class Router: verbose_router_logger.info( f"litellm.acompletion(model={model_name})\033[32m 200 OK\033[0m" ) + + # set used api_base in response + if hasattr(response, "_hidden_params"): + response._hidden_params["api_base"] = deployment.get( + "litellm_params", {} + ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( @@ -602,6 +613,12 @@ class Router: verbose_router_logger.info( f"litellm.aimage_generation(model={model_name})\033[32m 200 OK\033[0m" ) + + # set used api_base in response + if hasattr(response, "_hidden_params"): + response._hidden_params["api_base"] = deployment.get( + "litellm_params", {} + ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( @@ -694,6 +711,11 @@ class Router: verbose_router_logger.info( f"litellm.amoderation(model={model_name})\033[32m 200 OK\033[0m" ) + # set used api_base in response + if hasattr(response, "_hidden_params"): + response._hidden_params["api_base"] = deployment.get( + "litellm_params", {} + ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( @@ -825,6 +847,12 @@ class Router: verbose_router_logger.info( f"litellm.atext_completion(model={model_name})\033[32m 200 OK\033[0m" ) + + # set used api_base in response + if hasattr(response, "_hidden_params"): + response._hidden_params["api_base"] = deployment.get( + "litellm_params", {} + ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( @@ -954,6 +982,12 @@ class Router: verbose_router_logger.info( f"litellm.aembedding(model={model_name})\033[32m 200 OK\033[0m" ) + + # set used api_base in response + if hasattr(response, "_hidden_params"): + response._hidden_params["api_base"] = deployment.get( + "litellm_params", {} + ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( From 0bb45b33fd1050abeeb48ce3bcc173d604219b36 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 2 Mar 2024 11:06:03 -0800 Subject: [PATCH 2/4] (feat) send api_base --- litellm/proxy/utils.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 1cc52401a1e..948e686dd9b 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -64,6 +64,7 @@ class ProxyLogging: litellm.callbacks.append(self.max_parallel_request_limiter) litellm.callbacks.append(self.max_budget_limiter) litellm.callbacks.append(self.cache_control_check) + litellm.callbacks.append(self.response_taking_too_long_callback) for callback in litellm.callbacks: if callback not in litellm.input_callback: litellm.input_callback.append(callback) @@ -142,6 +143,30 @@ class ProxyLogging: raise e return data + async def response_taking_too_long_callback( + self, + kwargs, # kwargs to completion + completion_response, # response from completion + start_time, + end_time, # start/end time + ): + if self.alerting is None: + return + time_difference = end_time - start_time + # Convert the timedelta to float (in seconds) + time_difference_float = time_difference.total_seconds() + litellm_params = kwargs.get("litellm_params", {}) + api_base = litellm_params.get("api_base", "") + model = kwargs.get("model", "") + messages = kwargs.get("messages", "") + request_info = f"\nRequest Model: `{model}`\nAPI Base: `{api_base}`\nMessages: `{messages}`" + slow_message = f"`Responses are slow - {round(time_difference_float,2)}s response time > Alerting threshold: {self.alerting_threshold}s`" + if time_difference_float > self.alerting_threshold: + await self.alerting_handler( + message=slow_message + request_info, + level="Low", + ) + async def response_taking_too_long( self, start_time: Optional[float] = None, @@ -189,16 +214,6 @@ class ProxyLogging: level="Medium", ) - elif ( - type == "slow_response" and start_time is not None and end_time is not None - ): - slow_message = f"`Responses are slow - {round(end_time-start_time,2)}s response time > Alerting threshold: {self.alerting_threshold}s`" - if end_time - start_time > self.alerting_threshold: - await self.alerting_handler( - message=slow_message + request_info, - level="Low", - ) - async def budget_alerts( self, type: Literal[ From 127bc743b2b298a3b79e80ec453b78aab5ca6e79 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 2 Mar 2024 11:09:40 -0800 Subject: [PATCH 3/4] (feat) cleanup --- litellm/proxy/proxy_server.py | 42 ----------------------------------- 1 file changed, 42 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 869de6dde30..17db8c3ab4f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2138,14 +2138,6 @@ async def async_data_generator(response, user_api_key_dict): except Exception as e: yield f"data: {str(e)}\n\n" - ### ALERTING ### - end_time = time.time() - asyncio.create_task( - proxy_logging_obj.response_taking_too_long( - start_time=start_time, end_time=end_time, type="slow_response" - ) - ) - # Streaming is done, yield the [DONE] chunk done_message = "[DONE]" yield f"data: {done_message}\n\n" @@ -2494,14 +2486,6 @@ async def completion( headers=custom_headers, ) - ### ALERTING ### - end_time = time.time() - asyncio.create_task( - proxy_logging_obj.response_taking_too_long( - start_time=start_time, end_time=end_time, type="slow_response" - ) - ) - fastapi_response.headers["x-litellm-model-id"] = model_id return response except Exception as e: @@ -2700,14 +2684,6 @@ async def chat_completion( headers=custom_headers, ) - ### ALERTING ### - end_time = time.time() - asyncio.create_task( - proxy_logging_obj.response_taking_too_long( - start_time=start_time, end_time=end_time, type="slow_response" - ) - ) - fastapi_response.headers["x-litellm-model-id"] = model_id ### CALL HOOKS ### - modify outgoing data @@ -2915,12 +2891,6 @@ async def embeddings( ### ALERTING ### data["litellm_status"] = "success" # used for alerting - end_time = time.time() - asyncio.create_task( - proxy_logging_obj.response_taking_too_long( - start_time=start_time, end_time=end_time, type="slow_response" - ) - ) return response except Exception as e: @@ -3066,12 +3036,6 @@ async def image_generation( ### ALERTING ### data["litellm_status"] = "success" # used for alerting - end_time = time.time() - asyncio.create_task( - proxy_logging_obj.response_taking_too_long( - start_time=start_time, end_time=end_time, type="slow_response" - ) - ) return response except Exception as e: @@ -3225,12 +3189,6 @@ async def moderations( ### ALERTING ### data["litellm_status"] = "success" # used for alerting - end_time = time.time() - asyncio.create_task( - proxy_logging_obj.response_taking_too_long( - start_time=start_time, end_time=end_time, type="slow_response" - ) - ) return response except Exception as e: From 868a415aa0160985ee27850f0faef04e06b1dfb1 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 2 Mar 2024 11:12:09 -0800 Subject: [PATCH 4/4] Revert "(feat) track used api_base in response" This reverts commit 3aba9019ac27481361c3fa87956907f5c41dd4ca. --- litellm/router.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 59cbf43d56c..6f33d0b0d5c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -357,11 +357,6 @@ class Router: verbose_router_logger.info( f"litellm.completion(model={model_name})\033[32m 200 OK\033[0m" ) - # set used api_base in response - if hasattr(response, "_hidden_params"): - response._hidden_params["api_base"] = deployment.get( - "litellm_params", {} - ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( @@ -451,12 +446,6 @@ class Router: verbose_router_logger.info( f"litellm.acompletion(model={model_name})\033[32m 200 OK\033[0m" ) - - # set used api_base in response - if hasattr(response, "_hidden_params"): - response._hidden_params["api_base"] = deployment.get( - "litellm_params", {} - ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( @@ -613,12 +602,6 @@ class Router: verbose_router_logger.info( f"litellm.aimage_generation(model={model_name})\033[32m 200 OK\033[0m" ) - - # set used api_base in response - if hasattr(response, "_hidden_params"): - response._hidden_params["api_base"] = deployment.get( - "litellm_params", {} - ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( @@ -711,11 +694,6 @@ class Router: verbose_router_logger.info( f"litellm.amoderation(model={model_name})\033[32m 200 OK\033[0m" ) - # set used api_base in response - if hasattr(response, "_hidden_params"): - response._hidden_params["api_base"] = deployment.get( - "litellm_params", {} - ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( @@ -847,12 +825,6 @@ class Router: verbose_router_logger.info( f"litellm.atext_completion(model={model_name})\033[32m 200 OK\033[0m" ) - - # set used api_base in response - if hasattr(response, "_hidden_params"): - response._hidden_params["api_base"] = deployment.get( - "litellm_params", {} - ).get("api_base", None) return response except Exception as e: verbose_router_logger.info( @@ -982,12 +954,6 @@ class Router: verbose_router_logger.info( f"litellm.aembedding(model={model_name})\033[32m 200 OK\033[0m" ) - - # set used api_base in response - if hasattr(response, "_hidden_params"): - response._hidden_params["api_base"] = deployment.get( - "litellm_params", {} - ).get("api_base", None) return response except Exception as e: verbose_router_logger.info(