diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 2b28b925f05..f1f6febeca4 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -1086,6 +1086,46 @@ async def test_acompletion_caching_on_router_caching_groups(): asyncio.run(test_acompletion_caching_on_router_caching_groups()) ``` +## Alerting 🚨 + +Send alerts to slack / your webhook url for the following events +- LLM API Exceptions +- Slow LLM Responses + +Get a slack webhook url from https://api.slack.com/messaging/webhooks + +#### Usage +Initialize an `AlertingConfig` and pass it to `litellm.Router`. The following code will trigger an alert because `api_key=bad-key` which is invalid + +```python +from litellm.router import AlertingConfig +import litellm +import os + +router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "bad_key", + }, + } + ], + alerting_config= AlertingConfig( + alerting_threshold=10, # threshold for slow / hanging llm responses (in seconds). Defaults to 300 seconds + webhook_url= os.getenv("SLACK_WEBHOOK_URL") # webhook you want to send alerts to + ), +) +try: + await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) +except: + pass +``` + ## Track cost for Azure Deployments **Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index f033d99cdc7..c53daf36c31 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -68,11 +68,15 @@ class SlackAlertingCacheKeys(Enum): class SlackAlerting(CustomLogger): + """ + Class for sending Slack Alerts + """ + # Class variables or attributes def __init__( self, internal_usage_cache: Optional[DualCache] = None, - alerting_threshold: float = 300, + alerting_threshold: float = 300, # threshold for slow / hanging llm responses (in seconds) alerting: Optional[List] = [], alert_types: Optional[ List[ @@ -97,6 +101,7 @@ class SlackAlerting(CustomLogger): Dict ] = None, # if user wants to separate alerts to diff channels alerting_args={}, + default_webhook_url: Optional[str] = None, ): self.alerting_threshold = alerting_threshold self.alerting = alerting @@ -106,6 +111,7 @@ class SlackAlerting(CustomLogger): self.alert_to_webhook_url = alert_to_webhook_url self.is_running = False self.alerting_args = SlackAlertingArgs(**alerting_args) + self.default_webhook_url = default_webhook_url def update_values( self, @@ -307,7 +313,7 @@ class SlackAlerting(CustomLogger): except Exception as e: return 0 - async def send_daily_reports(self, router: litellm.Router) -> bool: + async def send_daily_reports(self, router) -> bool: """ Send a daily report on: - Top 5 deployments with most failed requests @@ -749,6 +755,8 @@ Model Info: and alert_type in self.alert_to_webhook_url ): slack_webhook_url = self.alert_to_webhook_url[alert_type] + elif self.default_webhook_url is not None: + slack_webhook_url = self.default_webhook_url else: slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL", None) @@ -805,8 +813,16 @@ Model Info: updated_at=litellm.utils.get_utc_datetime(), ) ) + if "llm_exceptions" in self.alert_types: + original_exception = kwargs.get("exception", None) - async def _run_scheduler_helper(self, llm_router: litellm.Router) -> bool: + await self.send_alert( + message="LLM API Failure - " + str(original_exception), + level="High", + alert_type="llm_exceptions", + ) + + async def _run_scheduler_helper(self, llm_router) -> bool: """ Returns: - True -> report sent @@ -848,7 +864,7 @@ Model Info: return report_sent_bool - async def _run_scheduled_daily_report(self, llm_router: Optional[litellm.Router]): + async def _run_scheduled_daily_report(self, llm_router: Optional[Any] = None): """ If 'daily_reports' enabled diff --git a/litellm/router.py b/litellm/router.py index 3f2bef4768f..ddb006d527e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -44,6 +44,7 @@ from litellm.types.router import ( updateDeployment, updateLiteLLMParams, RetryPolicy, + AlertingConfig, ) from litellm.integrations.custom_logger import CustomLogger @@ -103,6 +104,7 @@ class Router: ] = "simple-shuffle", routing_strategy_args: dict = {}, # just for latency-based routing semaphore: Optional[asyncio.Semaphore] = None, + alerting_config: Optional[AlertingConfig] = None, ) -> None: """ Initialize the Router class with the given parameters for caching, reliability, and routing strategy. @@ -131,7 +133,7 @@ class Router: cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1. routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle". routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}. - + alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None. Returns: Router: An instance of the litellm.Router class. @@ -316,6 +318,9 @@ class Router: self.model_group_retry_policy: Optional[Dict[str, RetryPolicy]] = ( model_group_retry_policy ) + self.alerting_config: Optional[AlertingConfig] = alerting_config + if self.alerting_config is not None: + self._initialize_alerting() def routing_strategy_init(self, routing_strategy: str, routing_strategy_args: dict): if routing_strategy == "least-busy": @@ -3320,6 +3325,23 @@ class Router: ): return retry_policy.ContentPolicyViolationErrorRetries + def _initialize_alerting(self): + from litellm.integrations.slack_alerting import SlackAlerting + + router_alerting_config: AlertingConfig = self.alerting_config + + _slack_alerting_logger = SlackAlerting( + alerting_threshold=router_alerting_config.alerting_threshold, + alerting=["slack"], + default_webhook_url=router_alerting_config.webhook_url, + ) + + litellm.callbacks.append(_slack_alerting_logger) + litellm.success_callback.append( + _slack_alerting_logger.response_taking_too_long_callback + ) + print("\033[94m\nInitialized Alerting for litellm.Router\033[0m\n") # noqa + def flush_cache(self): litellm.cache = None self.cache.flush_cache() diff --git a/litellm/tests/test_alerting.py b/litellm/tests/test_alerting.py index 3734c29d289..b3232cae163 100644 --- a/litellm/tests/test_alerting.py +++ b/litellm/tests/test_alerting.py @@ -18,6 +18,10 @@ from unittest.mock import patch, MagicMock from litellm.utils import get_api_base from litellm.caching import DualCache from litellm.integrations.slack_alerting import SlackAlerting, DeploymentMetrics +import unittest.mock +from unittest.mock import AsyncMock +import pytest +from litellm.router import AlertingConfig, Router @pytest.mark.parametrize( @@ -313,3 +317,45 @@ async def test_daily_reports_redis_cache_scheduler(): # second call - expect empty await slack_alerting._run_scheduler_helper(llm_router=router) + + +@pytest.mark.asyncio +@pytest.mark.skip(reason="Local test. Test if slack alerts are sent.") +async def test_send_llm_exception_to_slack(): + from litellm.router import AlertingConfig + + # on async success + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "bad_key", + }, + }, + { + "model_name": "gpt-5-good", + "litellm_params": { + "model": "gpt-3.5-turbo", + }, + }, + ], + alerting_config=AlertingConfig( + alerting_threshold=0.5, webhook_url=os.getenv("SLACK_WEBHOOK_URL") + ), + ) + try: + await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) + except: + pass + + await router.acompletion( + model="gpt-5-good", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) + + await asyncio.sleep(3) diff --git a/litellm/types/router.py b/litellm/types/router.py index 4a62a267ecc..6ab83cec2ae 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -346,3 +346,19 @@ class RetryPolicy(BaseModel): RateLimitErrorRetries: Optional[int] = None ContentPolicyViolationErrorRetries: Optional[int] = None InternalServerErrorRetries: Optional[int] = None + + +class AlertingConfig(BaseModel): + """ + Use this configure alerting for the router. Receive alerts on the following events + - LLM API Exceptions + - LLM Responses Too Slow + - LLM Requests Hanging + + Args: + webhook_url: str - webhook url for alerting, slack provides a webhook url to send alerts to + alerting_threshold: Optional[float] = None - threshold for slow / hanging llm responses (in seconds) + """ + + webhook_url: str + alerting_threshold: Optional[float] = 300