From 01da3b0a7aca8e1cca5dec82a6851707a977a616 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 13 May 2024 16:38:36 -0700 Subject: [PATCH 1/4] Add ability to customize slack report frequency via `SLACK_DAILY_REPORT_FREQUENCY` env var Addresses one thing that @taralika mentioned in https://github.com/BerriAI/litellm/issues/3597 --- litellm/integrations/slack_alerting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 7252967d9c0..f91cdb29d79 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -31,7 +31,8 @@ class LiteLLMBase(BaseModel): class SlackAlertingArgs(LiteLLMBase): - daily_report_frequency: int = 12 * 60 * 60 # 12 hours + default_daily_report_frequency: int = 12 * 60 * 60 # 12 hours + daily_report_frequency: int = os.getenv("SLACK_DAILY_REPORT_FREQUENCY", default_daily_report_frequency) report_check_interval: int = 5 * 60 # 5 minutes From bfec894f6b870a15a470c39a276cbedc17ab5bf6 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 13 May 2024 17:08:53 -0700 Subject: [PATCH 2/4] Cast SLACK_DAILY_REPORT_FREQUENCY to int --- litellm/integrations/slack_alerting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index f91cdb29d79..04e5a4d1b5a 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -32,7 +32,7 @@ class LiteLLMBase(BaseModel): class SlackAlertingArgs(LiteLLMBase): default_daily_report_frequency: int = 12 * 60 * 60 # 12 hours - daily_report_frequency: int = os.getenv("SLACK_DAILY_REPORT_FREQUENCY", default_daily_report_frequency) + daily_report_frequency: int = int(os.getenv("SLACK_DAILY_REPORT_FREQUENCY", default_daily_report_frequency)) report_check_interval: int = 5 * 60 # 5 minutes From 94f735359678554e59c532adf13778308612a559 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 13 May 2024 17:10:41 -0700 Subject: [PATCH 3/4] Document SLACK_DAILY_REPORT_FREQUENCY in `docs/my-website/docs/proxy/alerting.md` --- docs/my-website/docs/proxy/alerting.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md index 4275e0bf06f..68c89de6247 100644 --- a/docs/my-website/docs/proxy/alerting.md +++ b/docs/my-website/docs/proxy/alerting.md @@ -20,7 +20,8 @@ Get a slack webhook url from https://api.slack.com/messaging/webhooks ### Step 2: Update config.yaml -Let's save a bad key to our proxy +- Set `SLACK_WEBHOOK_URL` in your proxy env to enable Slack alerts. +- Just for testing purposes, let's save a bad key to our proxy. ```yaml model_list: @@ -33,13 +34,11 @@ general_settings: alerting: ["slack"] alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+ +environment_variables: + SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/<>/<>/<>" + SLACK_DAILY_REPORT_FREQUENCY: "86400" # 24 hours; Optional: defaults to 12 hours ``` -Set `SLACK_WEBHOOK_URL` in your proxy env - -```shell -SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/<>/<>/<>" -``` ### Step 3: Start proxy From 4645451140f24c99143d375335942f4e8a6ab680 Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Mon, 13 May 2024 21:05:59 -0700 Subject: [PATCH 4/4] Add info about daily reports to alerting.md --- docs/my-website/docs/proxy/alerting.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md index 68c89de6247..ea7bdc11013 100644 --- a/docs/my-website/docs/proxy/alerting.md +++ b/docs/my-website/docs/proxy/alerting.md @@ -1,14 +1,21 @@ # 🚨 Alerting Get alerts for: + - Hanging LLM api calls - Failed LLM api calls - Slow LLM api calls - Budget Tracking per key/user: - - When a User/Key crosses their Budget - - When a User/Key is 15% away from crossing their Budget + - When a User/Key crosses their Budget + - When a User/Key is 15% away from crossing their Budget - Failed db read/writes +As a bonus, you can also get "daily reports" posted to your slack channel. +These reports contain key metrics like: + +- Top 5 deployments with most failed requests +- Top 5 slowest deployments + ## Quick Start Set up a slack alert channel to receive alerts from proxy.