From 32f3e032e97beabf17b5595a72972eccce7b2640 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 7 May 2024 15:10:47 -0700 Subject: [PATCH 1/8] feat - send slack alerts litellm.router --- litellm/integrations/slack_alerting.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 5546f7c3378..b6e2f7ba562 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -796,6 +796,14 @@ Model Info: updated_at=litellm.utils.get_utc_datetime(), ) ) + if "llm_exceptions" in self.alert_types: + original_exception = kwargs.get("exception", None) + + await self.send_alert( + message="LLM API Failure - " + str(original_exception), + level="High", + alert_type="llm_exceptions", + ) async def _run_scheduler_helper(self, llm_router: litellm.Router) -> bool: """ From b1230dd9194f8daa7985c0771957cc4819899f5f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 7 May 2024 15:12:21 -0700 Subject: [PATCH 2/8] test - slack alerts on router --- litellm/tests/test_alerting.py | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/litellm/tests/test_alerting.py b/litellm/tests/test_alerting.py index 3734c29d289..06921d8d62d 100644 --- a/litellm/tests/test_alerting.py +++ b/litellm/tests/test_alerting.py @@ -313,3 +313,70 @@ async def test_daily_reports_redis_cache_scheduler(): # second call - expect empty await slack_alerting._run_scheduler_helper(llm_router=router) + + +@pytest.mark.asyncio +async def test_send_llm_exception(slack_alerting): + with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert: + litellm.callbacks = [slack_alerting] + + # on async success + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "bad_key", + }, + } + ] + ) + try: + await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) + except: + pass + + await asyncio.sleep(3) + + mock_send_alert.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.skip(reason="Local test. Test if slack alerts are sent.") +async def test_send_llm_exception_to_slack(): + from litellm.integrations.slack_alerting import SlackAlerting + + new_alerting = SlackAlerting( + alerting_threshold=0.00002, + alerting=["slack"], + alert_types=["llm_exceptions", "llm_requests_hanging", "llm_too_slow"], + ) + + litellm.callbacks = [new_alerting] + litellm.set_verbose = True + + # on async success + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-5", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "bad_key", + }, + } + ] + ) + try: + await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) + except: + pass + + await asyncio.sleep(3) From 5fd3b12d34993bbe4aa8974e7d207b90c1ad1358 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 7 May 2024 17:46:18 -0700 Subject: [PATCH 3/8] add router alerting type --- litellm/types/router.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/litellm/types/router.py b/litellm/types/router.py index 4a62a267ecc..d79fb2e2e1e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -346,3 +346,19 @@ class RetryPolicy(BaseModel): RateLimitErrorRetries: Optional[int] = None ContentPolicyViolationErrorRetries: Optional[int] = None InternalServerErrorRetries: Optional[int] = None + + +class RouterAlerting(BaseModel): + """ + Use this configure alerting for the router. Receive alerts on the following events + - LLM API Exceptions + - LLM Responses Too Slow + - LLM Requests Hanging + + Args: + webhook_url: Optional[str] = None - webhook url for alerting + alerting_threshold: Optional[float] = None - threhshold for slow / hanging llm responses (in seconds) + """ + + webhook_url: Optional[str] = None + alerting_threshold: Optional[float] = None From c08352a0ce278d47b7423576813e43997b5c9624 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 7 May 2024 18:03:04 -0700 Subject: [PATCH 4/8] router- initialize alerting --- litellm/router.py | 24 +++++++++++++++++++++++- litellm/types/router.py | 8 ++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 3f2bef4768f..ddb006d527e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -44,6 +44,7 @@ from litellm.types.router import ( updateDeployment, updateLiteLLMParams, RetryPolicy, + AlertingConfig, ) from litellm.integrations.custom_logger import CustomLogger @@ -103,6 +104,7 @@ class Router: ] = "simple-shuffle", routing_strategy_args: dict = {}, # just for latency-based routing semaphore: Optional[asyncio.Semaphore] = None, + alerting_config: Optional[AlertingConfig] = None, ) -> None: """ Initialize the Router class with the given parameters for caching, reliability, and routing strategy. @@ -131,7 +133,7 @@ class Router: cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1. routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle". routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}. - + alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None. Returns: Router: An instance of the litellm.Router class. @@ -316,6 +318,9 @@ class Router: self.model_group_retry_policy: Optional[Dict[str, RetryPolicy]] = ( model_group_retry_policy ) + self.alerting_config: Optional[AlertingConfig] = alerting_config + if self.alerting_config is not None: + self._initialize_alerting() def routing_strategy_init(self, routing_strategy: str, routing_strategy_args: dict): if routing_strategy == "least-busy": @@ -3320,6 +3325,23 @@ class Router: ): return retry_policy.ContentPolicyViolationErrorRetries + def _initialize_alerting(self): + from litellm.integrations.slack_alerting import SlackAlerting + + router_alerting_config: AlertingConfig = self.alerting_config + + _slack_alerting_logger = SlackAlerting( + alerting_threshold=router_alerting_config.alerting_threshold, + alerting=["slack"], + default_webhook_url=router_alerting_config.webhook_url, + ) + + litellm.callbacks.append(_slack_alerting_logger) + litellm.success_callback.append( + _slack_alerting_logger.response_taking_too_long_callback + ) + print("\033[94m\nInitialized Alerting for litellm.Router\033[0m\n") # noqa + def flush_cache(self): litellm.cache = None self.cache.flush_cache() diff --git a/litellm/types/router.py b/litellm/types/router.py index d79fb2e2e1e..f3fa893246e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -348,7 +348,7 @@ class RetryPolicy(BaseModel): InternalServerErrorRetries: Optional[int] = None -class RouterAlerting(BaseModel): +class AlertingConfig(BaseModel): """ Use this configure alerting for the router. Receive alerts on the following events - LLM API Exceptions @@ -356,9 +356,9 @@ class RouterAlerting(BaseModel): - LLM Requests Hanging Args: - webhook_url: Optional[str] = None - webhook url for alerting + webhook_url: str - webhook url for alerting, slack provides a webhook url to send alerts to alerting_threshold: Optional[float] = None - threhshold for slow / hanging llm responses (in seconds) """ - webhook_url: Optional[str] = None - alerting_threshold: Optional[float] = None + webhook_url: str + alerting_threshold: Optional[float] = 300 From e8053c3d0bd6fbceb40e8d0a5bb8a6b341a5c3b2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 7 May 2024 18:17:12 -0700 Subject: [PATCH 5/8] fix slack alerting --- litellm/integrations/slack_alerting.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index b6e2f7ba562..d974cfbd3d8 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -68,11 +68,15 @@ class SlackAlertingCacheKeys(Enum): class SlackAlerting(CustomLogger): + """ + Class for sending Slack Alerts + """ + # Class variables or attributes def __init__( self, internal_usage_cache: Optional[DualCache] = None, - alerting_threshold: float = 300, + alerting_threshold: float = 300, # threshold for slow / hanging llm responses (in seconds) alerting: Optional[List] = [], alert_types: Optional[ List[ @@ -97,6 +101,7 @@ class SlackAlerting(CustomLogger): Dict ] = None, # if user wants to separate alerts to diff channels alerting_args={}, + default_webhook_url: Optional[str] = None, ): self.alerting_threshold = alerting_threshold self.alerting = alerting @@ -106,6 +111,7 @@ class SlackAlerting(CustomLogger): self.alert_to_webhook_url = alert_to_webhook_url self.is_running = False self.alerting_args = SlackAlertingArgs(**alerting_args) + self.default_webhook_url = default_webhook_url def update_values( self, @@ -302,7 +308,7 @@ class SlackAlerting(CustomLogger): except Exception as e: return 0 - async def send_daily_reports(self, router: litellm.Router) -> bool: + async def send_daily_reports(self, router) -> bool: """ Send a daily report on: - Top 5 deployments with most failed requests @@ -740,6 +746,8 @@ Model Info: and alert_type in self.alert_to_webhook_url ): slack_webhook_url = self.alert_to_webhook_url[alert_type] + elif self.default_webhook_url is not None: + slack_webhook_url = self.default_webhook_url else: slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL", None) @@ -805,7 +813,7 @@ Model Info: alert_type="llm_exceptions", ) - async def _run_scheduler_helper(self, llm_router: litellm.Router) -> bool: + async def _run_scheduler_helper(self, llm_router) -> bool: """ Returns: - True -> report sent @@ -847,7 +855,7 @@ Model Info: return report_sent_bool - async def _run_scheduled_daily_report(self, llm_router: Optional[litellm.Router]): + async def _run_scheduled_daily_report(self, llm_router: Optional[Any] = None): """ If 'daily_reports' enabled From d46544d2bc83d87df66cd978f887be7c4ecf0b34 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 7 May 2024 18:26:45 -0700 Subject: [PATCH 6/8] docs setup alerting on router --- docs/my-website/docs/routing.md | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 2b28b925f05..f1f6febeca4 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -1086,6 +1086,46 @@ async def test_acompletion_caching_on_router_caching_groups(): asyncio.run(test_acompletion_caching_on_router_caching_groups()) ``` +## Alerting 🚨 + +Send alerts to slack / your webhook url for the following events +- LLM API Exceptions +- Slow LLM Responses + +Get a slack webhook url from https://api.slack.com/messaging/webhooks + +#### Usage +Initialize an `AlertingConfig` and pass it to `litellm.Router`. The following code will trigger an alert because `api_key=bad-key` which is invalid + +```python +from litellm.router import AlertingConfig +import litellm +import os + +router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + "api_key": "bad_key", + }, + } + ], + alerting_config= AlertingConfig( + alerting_threshold=10, # threshold for slow / hanging llm responses (in seconds). Defaults to 300 seconds + webhook_url= os.getenv("SLACK_WEBHOOK_URL") # webhook you want to send alerts to + ), +) +try: + await router.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) +except: + pass +``` + ## Track cost for Azure Deployments **Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking From dc742044276de5e85086b5fbc86ba996d6baf47c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 7 May 2024 18:27:49 -0700 Subject: [PATCH 7/8] fix typo --- litellm/types/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/types/router.py b/litellm/types/router.py index f3fa893246e..6ab83cec2ae 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -357,7 +357,7 @@ class AlertingConfig(BaseModel): Args: webhook_url: str - webhook url for alerting, slack provides a webhook url to send alerts to - alerting_threshold: Optional[float] = None - threhshold for slow / hanging llm responses (in seconds) + alerting_threshold: Optional[float] = None - threshold for slow / hanging llm responses (in seconds) """ webhook_url: str From 596adf6e2f313efb591d80c934cb7d988a4d300e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 7 May 2024 19:04:25 -0700 Subject: [PATCH 8/8] test - slack alerting on litellm router --- litellm/tests/test_alerting.py | 65 ++++++++++++---------------------- 1 file changed, 22 insertions(+), 43 deletions(-) diff --git a/litellm/tests/test_alerting.py b/litellm/tests/test_alerting.py index 06921d8d62d..b3232cae163 100644 --- a/litellm/tests/test_alerting.py +++ b/litellm/tests/test_alerting.py @@ -18,6 +18,10 @@ from unittest.mock import patch, MagicMock from litellm.utils import get_api_base from litellm.caching import DualCache from litellm.integrations.slack_alerting import SlackAlerting, DeploymentMetrics +import unittest.mock +from unittest.mock import AsyncMock +import pytest +from litellm.router import AlertingConfig, Router @pytest.mark.parametrize( @@ -315,61 +319,31 @@ async def test_daily_reports_redis_cache_scheduler(): await slack_alerting._run_scheduler_helper(llm_router=router) -@pytest.mark.asyncio -async def test_send_llm_exception(slack_alerting): - with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert: - litellm.callbacks = [slack_alerting] - - # on async success - router = litellm.Router( - model_list=[ - { - "model_name": "gpt-5", - "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": "bad_key", - }, - } - ] - ) - try: - await router.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hey, how's it going?"}], - ) - except: - pass - - await asyncio.sleep(3) - - mock_send_alert.assert_awaited_once() - - @pytest.mark.asyncio @pytest.mark.skip(reason="Local test. Test if slack alerts are sent.") async def test_send_llm_exception_to_slack(): - from litellm.integrations.slack_alerting import SlackAlerting - - new_alerting = SlackAlerting( - alerting_threshold=0.00002, - alerting=["slack"], - alert_types=["llm_exceptions", "llm_requests_hanging", "llm_too_slow"], - ) - - litellm.callbacks = [new_alerting] - litellm.set_verbose = True + from litellm.router import AlertingConfig # on async success router = litellm.Router( model_list=[ { - "model_name": "gpt-5", + "model_name": "gpt-3.5-turbo", "litellm_params": { "model": "gpt-3.5-turbo", "api_key": "bad_key", }, - } - ] + }, + { + "model_name": "gpt-5-good", + "litellm_params": { + "model": "gpt-3.5-turbo", + }, + }, + ], + alerting_config=AlertingConfig( + alerting_threshold=0.5, webhook_url=os.getenv("SLACK_WEBHOOK_URL") + ), ) try: await router.acompletion( @@ -379,4 +353,9 @@ async def test_send_llm_exception_to_slack(): except: pass + await router.acompletion( + model="gpt-5-good", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) + await asyncio.sleep(3)