From da0e5d1b8d08e99791174628308190ccfebae25e Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 20 May 2024 15:30:56 -0700
Subject: [PATCH 1/6] feat(slack_alerting.py): support webhook for budget
 alerts

Closes https://github.com/BerriAI/litellm/issues/3743
---
 litellm/integrations/slack_alerting.py  | 138 ++++++++++++++++--------
 litellm/proxy/_super_secret_config.yaml |   8 +-
 litellm/proxy/_types.py                 |  13 +++
 litellm/proxy/proxy_server.py           | 107 ++++++++++++------
 litellm/proxy/utils.py                  |  20 +++-
 litellm/tests/test_alerting.py          |  70 +++++++++---
 6 files changed, 255 insertions(+), 101 deletions(-)

diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py
index 227db079db6..c1e2735ddf2 100644
--- a/litellm/integrations/slack_alerting.py
+++ b/litellm/integrations/slack_alerting.py
@@ -1,7 +1,7 @@
 #### What this does ####
 #    Class for sending Slack Alerts #
 import dotenv, os
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy._types import UserAPIKeyAuth, CallInfo
 from litellm._logging import verbose_logger, verbose_proxy_logger
 import litellm, threading
 from typing import List, Literal, Any, Union, Optional, Dict
@@ -571,20 +571,35 @@ class SlackAlerting(CustomLogger):
                     alert_type="llm_requests_hanging",
                 )
 
+    async def failed_tracking_alert(self, error_message: str):
+        """Raise alert when tracking failed for specific model"""
+        _cache: DualCache = self.internal_usage_cache
+        message = "Failed Tracking Cost for" + error_message
+        _cache_key = "budget_alerts:failed_tracking:{}".format(message)
+        result = await _cache.async_get_cache(key=_cache_key)
+        if result is None:
+            await self.send_alert(
+                message=message, level="High", alert_type="budget_alerts"
+            )
+            await _cache.async_set_cache(
+                key=_cache_key,
+                value="SENT",
+                ttl=self.alerting_args.budget_alert_ttl,
+            )
+
     async def budget_alerts(
         self,
         type: Literal[
             "token_budget",
             "user_budget",
+            "team_budget",
             "user_and_proxy_budget",
             "failed_budgets",
-            "failed_tracking",
             "projected_limit_exceeded",
         ],
         user_max_budget: float,
         user_current_spend: float,
-        user_info=None,
-        error_message="",
+        user_info: CallInfo,
     ):
         ## PREVENTITIVE ALERTING ## - https://github.com/BerriAI/litellm/issues/2727
         # - Alert once within 24hr period
@@ -598,39 +613,23 @@ class SlackAlerting(CustomLogger):
         if "budget_alerts" not in self.alert_types:
             return
         _id: str = "default_id"  # used for caching
+        user_info_str = ""
         if type == "user_and_proxy_budget":
-            user_info = dict(user_info)
-            user_id = user_info["user_id"]
-            _id = user_id
-            max_budget = user_info["max_budget"]
-            spend = user_info["spend"]
-            user_email = user_info["user_email"]
-            user_info = f"""\nUser ID: {user_id}\nMax Budget: ${max_budget}\nSpend: ${spend}\nUser Email: {user_email}"""
-        elif type == "token_budget":
-            token_info = dict(user_info)
-            token = token_info["token"]
+            user_id = user_info.user_id
+            if user_id is not None:
+                _id = user_id
+            max_budget = user_info.max_budget
+            spend = user_info.spend
+            user_email = user_info.user_email
+            user_info_str = f"""\nUser ID: {user_id}\nMax Budget: ${max_budget}\nSpend: ${spend}\nUser Email: {user_email}"""
+        elif type == "token_budget" or type == "team_budget":
+            token_info = user_info
+            token = token_info.token
             _id = token
-            spend = token_info["spend"]
-            max_budget = token_info["max_budget"]
-            user_id = token_info["user_id"]
-            user_info = f"""\nToken: {token}\nSpend: ${spend}\nMax Budget: ${max_budget}\nUser ID: {user_id}"""
-        elif type == "failed_tracking":
-            user_id = str(user_info)
-            _id = user_id
-            user_info = f"\nUser ID: {user_id}\n Error {error_message}"
-            message = "Failed Tracking Cost for" + user_info
-            _cache_key = "budget_alerts:failed_tracking:{}".format(_id)
-            result = await _cache.async_get_cache(key=_cache_key)
-            if result is None:
-                await self.send_alert(
-                    message=message, level="High", alert_type="budget_alerts"
-                )
-                await _cache.async_set_cache(
-                    key=_cache_key,
-                    value="SENT",
-                    ttl=self.alerting_args.budget_alert_ttl,
-                )
-            return
+            spend = token_info.spend
+            max_budget = token_info.max_budget
+            user_id = token_info.user_id
+            user_info_str = f"""\nToken: {token}\nSpend: ${spend}\nMax Budget: ${max_budget}\nUser ID: {user_id}"""
         elif type == "projected_limit_exceeded" and user_info is not None:
             """
             Input variables:
@@ -642,12 +641,15 @@ class SlackAlerting(CustomLogger):
             user_max_budget=soft_limit,
             user_current_spend=new_spend
             """
-            message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` {user_info["key_alias"]} \n`Expected Day of Error`: {user_info["projected_exceeded_date"]} \n`Current Spend`: {user_current_spend} \n`Projected Spend at end of month`: {user_info["projected_spend"]} \n`Soft Limit`: {user_max_budget}"""
+            message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` {user_info.key_alias} \n`Expected Day of Error`: {user_info.projected_exceeded_data} \n`Current Spend`: {user_current_spend} \n`Projected Spend at end of month`: {user_info.projected_spend} \n`Soft Limit`: {user_max_budget}"""
             _cache_key = "budget_alerts:projected_limit_exceeded:{}".format(_id)
             result = await _cache.async_get_cache(key=_cache_key)
             if result is None:
                 await self.send_alert(
-                    message=message, level="High", alert_type="budget_alerts"
+                    message=message,
+                    level="High",
+                    alert_type="budget_alerts",
+                    user_info=user_info,
                 )
                 await _cache.async_set_cache(
                     key=_cache_key,
@@ -655,8 +657,6 @@ class SlackAlerting(CustomLogger):
                     ttl=self.alerting_args.budget_alert_ttl,
                 )
             return
-        else:
-            user_info = str(user_info)
 
         # percent of max_budget left to spend
         if user_max_budget > 0:
@@ -664,18 +664,21 @@ class SlackAlerting(CustomLogger):
         else:
             percent_left = 0
         verbose_proxy_logger.debug(
-            f"Budget Alerts: Percent left: {percent_left} for {user_info}"
+            f"Budget Alerts: Percent left: {percent_left} for {user_info_str}"
         )
 
         # check if crossed budget
         if user_current_spend >= user_max_budget:
-            verbose_proxy_logger.debug("Budget Crossed for %s", user_info)
-            message = "Budget Crossed for" + user_info
+            verbose_proxy_logger.debug("Budget Crossed for %s", user_info_str)
+            message = "Budget Crossed for" + user_info_str
             _cache_key = "budget_alerts:budget_crossed:{}".format(_id)
             result = await _cache.async_get_cache(key=_cache_key)
             if result is None:
                 await self.send_alert(
-                    message=message, level="High", alert_type="budget_alerts"
+                    message=message,
+                    level="High",
+                    alert_type="budget_alerts",
+                    user_info=user_info,
                 )
                 await _cache.async_set_cache(
                     key=_cache_key,
@@ -686,7 +689,7 @@ class SlackAlerting(CustomLogger):
 
         # check if 5% of max budget is left
         if percent_left <= 0.05:
-            message = "5% budget left for" + user_info
+            message = "5% budget left for" + user_info_str
             _cache_key = "budget_alerts:5_perc_budget_crossed:{}".format(_id)
             result = await _cache.async_get_cache(key=_cache_key)
             if result is None:
@@ -694,6 +697,7 @@ class SlackAlerting(CustomLogger):
                     message=message,
                     level="Medium",
                     alert_type="budget_alerts",
+                    user_info=user_info,
                 )
 
                 await _cache.async_set_cache(
@@ -706,7 +710,7 @@ class SlackAlerting(CustomLogger):
 
         # check if 15% of max budget is left
         if percent_left <= 0.15:
-            message = "15% budget left for" + user_info
+            message = "15% budget left for" + user_info_str
             _cache_key = "budget_alerts:15_perc_budget_crossed:{}".format(_id)
             result = await _cache.async_get_cache(key=_cache_key)
             if result is None:
@@ -714,6 +718,7 @@ class SlackAlerting(CustomLogger):
                     message=message,
                     level="Low",
                     alert_type="budget_alerts",
+                    user_info=user_info,
                 )
                 await _cache.async_set_cache(
                     key=_cache_key,
@@ -780,6 +785,36 @@ Model Info:
     async def model_removed_alert(self, model_name: str):
         pass
 
+    async def send_webhook_alert(
+        self, alert_type: Literal["budget_alerts"], call_info: CallInfo
+    ) -> bool:
+        """
+        Sends structured alert to webhook, if set.
+
+        Currently only implemented for budget alerts
+
+        Returns -> True if sent, False if not.
+        """
+
+        webhook_url = os.getenv("WEBHOOK_URL", None)
+        if webhook_url is None:
+            raise Exception("Missing webhook_url from environment")
+
+        payload = call_info.model_dump_json()
+        headers = {"Content-type": "application/json"}
+
+        response = await self.async_http_handler.post(
+            url=webhook_url,
+            headers=headers,
+            data=payload,
+        )
+        if response.status_code == 200:
+            return True
+        else:
+            print("Error sending webhook alert. Error=", response.text)  # noqa
+
+        return False
+
     async def send_alert(
         self,
         message: str,
@@ -795,6 +830,7 @@ Model Info:
             "new_model_added",
             "cooldown_deployment",
         ],
+        user_info: Optional[CallInfo] = None,
         **kwargs,
     ):
         """
@@ -814,6 +850,18 @@ Model Info:
         if self.alerting is None:
             return
 
+        if (
+            "webhook" in self.alerting
+            and alert_type == "budget_alerts"
+            and user_info is not None
+        ):
+            await self.send_webhook_alert(
+                alert_type="budget_alerts", call_info=user_info
+            )
+
+        if "slack" not in self.alerting:
+            return
+
         if alert_type not in self.alert_types:
             return
 
diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml
index 2195a077d36..4277fbacb98 100644
--- a/litellm/proxy/_super_secret_config.yaml
+++ b/litellm/proxy/_super_secret_config.yaml
@@ -17,4 +17,10 @@ model_list:
       api_key: os.environ/AZURE_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault
   
 router_settings:
-  enable_pre_call_checks: true
\ No newline at end of file
+  enable_pre_call_checks: true
+
+general_settings: 
+  alerting: ["slack", "webhook"]
+
+environment_variables: 
+  WEBHOOK_URL: http://0.0.0.0:8000
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index b900b623be3..443e603c25a 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -1026,3 +1026,16 @@ class TokenCountResponse(LiteLLMBase):
     request_model: str
     model_used: str
     tokenizer_type: str
+
+
+class CallInfo(LiteLLMBase):
+    """Used for slack budget alerting"""
+
+    token: str = Field(description="Hashed value of that key")
+    spend: float = Field(description="The spend for that key")
+    max_budget: Optional[float] = None
+    user_id: Optional[str] = None
+    user_email: Optional[str] = None
+    key_alias: Optional[str] = None
+    projected_exceeded_data: Optional[str] = None
+    projected_spend: Optional[float] = None
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 0deb20f74a6..a36e2916341 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -589,12 +589,12 @@ async def user_api_key_auth(
                             ttl=UserAPIKeyCacheTTLEnum.global_proxy_spend.value,
                         )
                     if global_proxy_spend is not None:
-                        user_info = {
-                            "user_id": litellm_proxy_admin_name,
-                            "max_budget": litellm.max_budget,
-                            "spend": global_proxy_spend,
-                            "user_email": "",
-                        }
+                        user_info = CallInfo(
+                            user_id=litellm_proxy_admin_name,
+                            max_budget=litellm.max_budget,
+                            spend=global_proxy_spend,
+                            token=valid_token["token"],
+                        )
                         asyncio.create_task(
                             proxy_logging_obj.budget_alerts(
                                 user_max_budget=litellm.max_budget,
@@ -922,12 +922,19 @@ async def user_api_key_auth(
                                 user_max_budget is not None
                                 and user_current_spend is not None
                             ):
+                                call_info = CallInfo(
+                                    token=valid_token["token"],
+                                    spend=valid_token["spend"],
+                                    user_id=_user.get("user_id", None),
+                                    user_email=_user.get("user_email", None),
+                                    key_alias=valid_token["key_alias"],
+                                )
                                 asyncio.create_task(
                                     proxy_logging_obj.budget_alerts(
                                         user_max_budget=user_max_budget,
                                         user_current_spend=user_current_spend,
                                         type="user_and_proxy_budget",
-                                        user_info=_user,
+                                        user_info=call_info,
                                     )
                                 )
 
@@ -947,12 +954,22 @@ async def user_api_key_auth(
                             user_max_budget is not None
                             and user_current_spend is not None
                         ):
+                            call_info = CallInfo(
+                                token=valid_token["token"],
+                                spend=valid_token["spend"],
+                                max_budget=valid_token["max_budget"],
+                                user_id=getattr(user_id_information, "user_id", None),
+                                user_email=getattr(
+                                    user_id_information, "user_email", None
+                                ),
+                                key_alias=valid_token["key_alias"],
+                            )
                             asyncio.create_task(
                                 proxy_logging_obj.budget_alerts(
                                     user_max_budget=user_max_budget,
                                     user_current_spend=user_current_spend,
                                     type="user_budget",
-                                    user_info=user_id_information,
+                                    user_info=call_info,
                                 )
                             )
 
@@ -982,12 +999,18 @@ async def user_api_key_auth(
 
             # Check 4. Token Spend is under budget
             if valid_token.spend is not None and valid_token.max_budget is not None:
+                call_info = CallInfo(
+                    token=valid_token["token"],
+                    spend=valid_token["spend"],
+                    max_budget=valid_token["max_budget"],
+                    user_id=valid_token["user_id"],
+                )
                 asyncio.create_task(
                     proxy_logging_obj.budget_alerts(
                         user_max_budget=valid_token.max_budget,
                         user_current_spend=valid_token.spend,
                         type="token_budget",
-                        user_info=valid_token,
+                        user_info=call_info,
                     )
                 )
 
@@ -1038,12 +1061,18 @@ async def user_api_key_auth(
                 and hasattr(valid_token, "team_max_budget")
                 and valid_token.team_max_budget is not None
             ):
+                call_info = CallInfo(
+                    token=valid_token["token"],
+                    spend=valid_token["spend"],
+                    max_budget=valid_token["max_budget"],
+                    user_id=valid_token["user_id"],
+                )
                 asyncio.create_task(
                     proxy_logging_obj.budget_alerts(
                         user_max_budget=valid_token.team_max_budget,
                         user_current_spend=valid_token.spend,
-                        type="token_budget",
-                        user_info=valid_token,
+                        type="team_budget",
+                        user_info=call_info,
                     )
                 )
 
@@ -1059,12 +1088,18 @@ async def user_api_key_auth(
                 and hasattr(valid_token, "team_max_budget")
                 and valid_token.team_max_budget is not None
             ):
+                call_info = CallInfo(
+                    token=valid_token["token"],
+                    spend=valid_token["spend"],
+                    max_budget=valid_token["max_budget"],
+                    user_id=valid_token["user_id"],
+                )
                 asyncio.create_task(
                     proxy_logging_obj.budget_alerts(
                         user_max_budget=valid_token.team_max_budget,
                         user_current_spend=valid_token.team_spend,
-                        type="token_budget",
-                        user_info=valid_token,
+                        type="team_budget",
+                        user_info=call_info,
                     )
                 )
 
@@ -1110,18 +1145,18 @@ async def user_api_key_auth(
                     )
 
                 if global_proxy_spend is not None:
-                    user_info = {
-                        "user_id": litellm_proxy_admin_name,
-                        "max_budget": litellm.max_budget,
-                        "spend": global_proxy_spend,
-                        "user_email": "",
-                    }
+                    call_info = CallInfo(
+                        token=valid_token["token"],
+                        spend=global_proxy_spend,
+                        max_budget=litellm.max_budget,
+                        user_id=litellm_proxy_admin_name,
+                    )
                     asyncio.create_task(
                         proxy_logging_obj.budget_alerts(
                             user_max_budget=litellm.max_budget,
                             user_current_spend=global_proxy_spend,
                             type="user_and_proxy_budget",
-                            user_info=user_info,
+                            user_info=call_info,
                         )
                     )
             _ = common_checks(
@@ -1513,13 +1548,8 @@ async def _PROXY_track_cost_callback(
         model = kwargs.get("model", "")
         metadata = kwargs.get("litellm_params", {}).get("metadata", {})
         error_msg += f"\n Args to _PROXY_track_cost_callback\n model: {model}\n metadata: {metadata}\n"
-        user_id = user_id or "not-found"
         asyncio.create_task(
-            proxy_logging_obj.budget_alerts(
-                user_max_budget=0,
-                user_current_spend=0,
-                type="failed_tracking",
-                user_info=user_id,
+            proxy_logging_obj.failed_tracking_alert(
                 error_message=error_msg,
             )
         )
@@ -1766,22 +1796,29 @@ async def update_cache(
                 == True
             )
         ):
-            key_alias = existing_spend_obj.key_alias
             projected_spend, projected_exceeded_date = _get_projected_spend_over_limit(
                 current_spend=new_spend,
-                soft_budget_limit=existing_spend_obj.litellm_budget_table.soft_budget,
+                soft_budget_limit=existing_spend_obj.litellm_budget_table.get(
+                    "soft_budget", None
+                ),
+            )  # type: ignore
+            soft_limit = existing_spend_obj.litellm_budget_table.get(
+                "soft_budget", float("inf")
+            )
+            call_info = CallInfo(
+                token=existing_spend_obj.token or "",
+                spend=existing_spend_obj.spend,
+                key_alias=existing_spend_obj.key_alias,
+                max_budget=existing_spend_obj.max_budget,
+                user_id=existing_spend_obj.user_id,
+                projected_spend=projected_spend,
+                projected_exceeded_data=projected_exceeded_date,
             )
-            soft_limit = existing_spend_obj.litellm_budget_table.soft_budget
-            user_info = {
-                "key_alias": key_alias,
-                "projected_spend": projected_spend,
-                "projected_exceeded_date": projected_exceeded_date,
-            }
             # alert user
             asyncio.create_task(
                 proxy_logging_obj.budget_alerts(
                     type="projected_limit_exceeded",
-                    user_info=user_info,
+                    user_info=call_info,
                     user_max_budget=soft_limit,
                     user_current_spend=new_spend,
                 )
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 09e772e10b1..4bd793a496c 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -11,6 +11,7 @@ from litellm.proxy._types import (
     LiteLLM_EndUserTable,
     LiteLLM_TeamTable,
     Member,
+    CallInfo,
 )
 from litellm.caching import DualCache, RedisCache
 from litellm.router import Deployment, ModelInfo, LiteLLM_Params
@@ -276,20 +277,26 @@ class ProxyLogging:
                 raise e
         return data
 
+    async def failed_tracking_alert(self, error_message: str):
+        if self.alerting is None:
+            return
+        await self.slack_alerting_instance.failed_tracking_alert(
+            error_message=error_message
+        )
+
     async def budget_alerts(
         self,
         type: Literal[
             "token_budget",
             "user_budget",
+            "team_budget",
             "user_and_proxy_budget",
             "failed_budgets",
-            "failed_tracking",
             "projected_limit_exceeded",
         ],
         user_max_budget: float,
         user_current_spend: float,
-        user_info=None,
-        error_message="",
+        user_info: CallInfo,
     ):
         if self.alerting is None:
             # do nothing if alerting is not switched on
@@ -299,7 +306,6 @@ class ProxyLogging:
             user_max_budget=user_max_budget,
             user_current_spend=user_current_spend,
             user_info=user_info,
-            error_message=error_message,
         )
 
     async def alerting_handler(
@@ -355,7 +361,11 @@ class ProxyLogging:
         for client in self.alerting:
             if client == "slack":
                 await self.slack_alerting_instance.send_alert(
-                    message=message, level=level, alert_type=alert_type, **extra_kwargs
+                    message=message,
+                    level=level,
+                    alert_type=alert_type,
+                    user_info=None,
+                    **extra_kwargs,
                 )
             elif client == "sentry":
                 if litellm.utils.sentry_sdk_instance is not None:
diff --git a/litellm/tests/test_alerting.py b/litellm/tests/test_alerting.py
index 4431f835a7f..7d277f3e5f3 100644
--- a/litellm/tests/test_alerting.py
+++ b/litellm/tests/test_alerting.py
@@ -1,7 +1,7 @@
 # What is this?
 ## Tests slack alerting on proxy logging object
 
-import sys, json, uuid
+import sys, json, uuid, random
 import os
 import io, asyncio
 from datetime import datetime, timedelta
@@ -22,6 +22,7 @@ import unittest.mock
 from unittest.mock import AsyncMock
 import pytest
 from litellm.router import AlertingConfig, Router
+from litellm.proxy._types import CallInfo
 
 
 @pytest.mark.parametrize(
@@ -417,9 +418,9 @@ async def test_send_daily_reports_all_zero_or_none():
     [
         "token_budget",
         "user_budget",
+        "team_budget",
         "user_and_proxy_budget",
         "failed_budgets",
-        "failed_tracking",
         "projected_limit_exceeded",
     ],
 )
@@ -428,19 +429,18 @@ async def test_send_token_budget_crossed_alerts(alerting_type):
     slack_alerting = SlackAlerting()
 
     with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
-        if alerting_type == "failed_tracking":
-            user_info = "ishaan@berri.ai"
-        else:
-            user_info = {
-                "token": "50e55ca5bfbd0759697538e8d23c0cd5031f52d9e19e176d7233b20c7c4d3403",
-                "spend": uuid.uuid4(),
-                "max_budget": None,
-                "user_id": "ishaan@berri.ai",
-                "user_email": "ishaan@berri.ai",
-                "key_alias": "my-test-key",
-                "projected_exceeded_date": "10/20/2024",
-                "projected_spend": 200,
-            }
+        user_info = {
+            "token": "50e55ca5bfbd0759697538e8d23c0cd5031f52d9e19e176d7233b20c7c4d3403",
+            "spend": random.uniform(1.5, 1.9),
+            "max_budget": None,
+            "user_id": "ishaan@berri.ai",
+            "user_email": "ishaan@berri.ai",
+            "key_alias": "my-test-key",
+            "projected_exceeded_date": "10/20/2024",
+            "projected_spend": 200,
+        }
+
+        user_info = CallInfo(**user_info)
 
         for _ in range(50):
             await slack_alerting.budget_alerts(
@@ -450,3 +450,43 @@ async def test_send_token_budget_crossed_alerts(alerting_type):
                 user_max_budget=100,
             )
         mock_send_alert.assert_awaited_once()
+
+
+@pytest.mark.parametrize(
+    "alerting_type",
+    [
+        "token_budget",
+        "user_budget",
+        "team_budget",
+        "user_and_proxy_budget",
+        "failed_budgets",
+        "projected_limit_exceeded",
+    ],
+)
+@pytest.mark.asyncio
+async def test_webhook_alerting(alerting_type):
+    slack_alerting = SlackAlerting(alerting=["webhook"])
+
+    with patch.object(
+        slack_alerting, "send_webhook_alert", new=AsyncMock()
+    ) as mock_send_alert:
+        user_info = {
+            "token": "50e55ca5bfbd0759697538e8d23c0cd5031f52d9e19e176d7233b20c7c4d3403",
+            "spend": random.uniform(1.5, 1.9),
+            "max_budget": None,
+            "user_id": "ishaan@berri.ai",
+            "user_email": "ishaan@berri.ai",
+            "key_alias": "my-test-key",
+            "projected_exceeded_date": "10/20/2024",
+            "projected_spend": 200,
+        }
+
+        user_info = CallInfo(**user_info)
+        for _ in range(50):
+            await slack_alerting.budget_alerts(
+                type=alerting_type,
+                user_info=user_info,
+                user_current_spend=86,
+                user_max_budget=100,
+            )
+        mock_send_alert.assert_awaited_once()

From cba2b729b26331b381ce4a68ae4adda827bdb742 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 20 May 2024 15:37:35 -0700
Subject: [PATCH 2/6] test(test_alerting.py): testing fixes

---
 litellm/tests/test_alerting.py | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/litellm/tests/test_alerting.py b/litellm/tests/test_alerting.py
index 7d277f3e5f3..b1887f73d27 100644
--- a/litellm/tests/test_alerting.py
+++ b/litellm/tests/test_alerting.py
@@ -124,7 +124,9 @@ from datetime import datetime, timedelta
 
 @pytest.fixture
 def slack_alerting():
-    return SlackAlerting(alerting_threshold=1, internal_usage_cache=DualCache())
+    return SlackAlerting(
+        alerting_threshold=1, internal_usage_cache=DualCache(), alerting=["slack"]
+    )
 
 
 # Test for hanging LLM responses
@@ -162,7 +164,10 @@ async def test_budget_alerts_crossed(slack_alerting):
     user_current_spend = 101
     with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
         await slack_alerting.budget_alerts(
-            "user_budget", user_max_budget, user_current_spend
+            "user_budget",
+            user_max_budget,
+            user_current_spend,
+            user_info=CallInfo(token="", spend=0),
         )
         mock_send_alert.assert_awaited_once()
 
@@ -174,12 +179,18 @@ async def test_budget_alerts_crossed_again(slack_alerting):
     user_current_spend = 101
     with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
         await slack_alerting.budget_alerts(
-            "user_budget", user_max_budget, user_current_spend
+            "user_budget",
+            user_max_budget,
+            user_current_spend,
+            user_info=CallInfo(token="", spend=0),
         )
         mock_send_alert.assert_awaited_once()
         mock_send_alert.reset_mock()
         await slack_alerting.budget_alerts(
-            "user_budget", user_max_budget, user_current_spend
+            "user_budget",
+            user_max_budget,
+            user_current_spend,
+            user_info=CallInfo(token="", spend=0),
         )
         mock_send_alert.assert_not_awaited()
 

From 867f9300e3373744dd788d58ef44d661ce476313 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 20 May 2024 16:55:01 -0700
Subject: [PATCH 3/6] fix(slack_alerting.py): cleanup webhook event

---
 litellm/integrations/slack_alerting.py  | 171 +++++++++---------------
 litellm/proxy/_super_secret_config.yaml |   2 +-
 litellm/proxy/_types.py                 |   7 +-
 litellm/proxy/proxy_server.py           | 129 ++++++++----------
 litellm/proxy/utils.py                  |   7 +-
 litellm/tests/test_alerting.py          |   2 -
 6 files changed, 129 insertions(+), 189 deletions(-)

diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py
index c1e2735ddf2..6c12f0a7d40 100644
--- a/litellm/integrations/slack_alerting.py
+++ b/litellm/integrations/slack_alerting.py
@@ -39,6 +39,12 @@ class SlackAlertingArgs(LiteLLMBase):
     budget_alert_ttl: int = 24 * 60 * 60  # 24 hours
 
 
+class WebhookEvent(CallInfo):
+    event: Literal["budget_crossed", "threshold_crossed", "projected_limit_exceeded"]
+    event_group: Literal["user", "key", "team", "proxy"]
+    event_message: str  # human-readable description of event
+
+
 class DeploymentMetrics(LiteLLMBase):
     """
     Metrics per deployment, stored in cache
@@ -593,12 +599,9 @@ class SlackAlerting(CustomLogger):
             "token_budget",
             "user_budget",
             "team_budget",
-            "user_and_proxy_budget",
-            "failed_budgets",
+            "proxy_budget",
             "projected_limit_exceeded",
         ],
-        user_max_budget: float,
-        user_current_spend: float,
         user_info: CallInfo,
     ):
         ## PREVENTITIVE ALERTING ## - https://github.com/BerriAI/litellm/issues/2727
@@ -613,118 +616,78 @@ class SlackAlerting(CustomLogger):
         if "budget_alerts" not in self.alert_types:
             return
         _id: str = "default_id"  # used for caching
-        user_info_str = ""
-        if type == "user_and_proxy_budget":
-            user_id = user_info.user_id
-            if user_id is not None:
-                _id = user_id
-            max_budget = user_info.max_budget
-            spend = user_info.spend
-            user_email = user_info.user_email
-            user_info_str = f"""\nUser ID: {user_id}\nMax Budget: ${max_budget}\nSpend: ${spend}\nUser Email: {user_email}"""
-        elif type == "token_budget" or type == "team_budget":
-            token_info = user_info
-            token = token_info.token
-            _id = token
-            spend = token_info.spend
-            max_budget = token_info.max_budget
-            user_id = token_info.user_id
-            user_info_str = f"""\nToken: {token}\nSpend: ${spend}\nMax Budget: ${max_budget}\nUser ID: {user_id}"""
-        elif type == "projected_limit_exceeded" and user_info is not None:
-            """
-            Input variables:
-            user_info = {
-                "key_alias": key_alias,
-                "projected_spend": projected_spend,
-                "projected_exceeded_date": projected_exceeded_date,
-            }
-            user_max_budget=soft_limit,
-            user_current_spend=new_spend
-            """
-            message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` {user_info.key_alias} \n`Expected Day of Error`: {user_info.projected_exceeded_data} \n`Current Spend`: {user_current_spend} \n`Projected Spend at end of month`: {user_info.projected_spend} \n`Soft Limit`: {user_max_budget}"""
-            _cache_key = "budget_alerts:projected_limit_exceeded:{}".format(_id)
-            result = await _cache.async_get_cache(key=_cache_key)
-            if result is None:
-                await self.send_alert(
-                    message=message,
-                    level="High",
-                    alert_type="budget_alerts",
-                    user_info=user_info,
-                )
-                await _cache.async_set_cache(
-                    key=_cache_key,
-                    value="SENT",
-                    ttl=self.alerting_args.budget_alert_ttl,
-                )
-            return
+        user_info_json = user_info.model_dump(exclude_none=True)
+        for k, v in user_info_json.items():
+            user_info_str = "\n{}: {}\n".format(k, v)
+
+        event: Optional[
+            Literal["budget_crossed", "threshold_crossed", "projected_limit_exceeded"]
+        ] = None
+        event_group: Optional[Literal["user", "team", "key", "proxy"]] = None
+        event_message: str = ""
+        webhook_event: Optional[WebhookEvent] = None
+        if type == "proxy_budget":
+            event_group = "proxy"
+            event_message += "Proxy Budget: "
+        elif type == "user_budget":
+            event_group = "user"
+            event_message += "User Budget: "
+            _id = user_info.user_id or _id
+        elif type == "team_budget":
+            event_group = "team"
+            event_message += "Team Budget: "
+            _id = user_info.team_id or _id
+        elif type == "token_budget":
+            event_group = "key"
+            event_message += "Key Budget: "
+            _id = user_info.token
+        elif type == "projected_limit_exceeded":
+            event_group = "key"
+            event_message += "Key Budget: Projected Limit Exceeded"
+            event = "projected_limit_exceeded"
+            _id = user_info.token
 
         # percent of max_budget left to spend
-        if user_max_budget > 0:
-            percent_left = (user_max_budget - user_current_spend) / user_max_budget
+        if user_info.max_budget > 0:
+            percent_left = (
+                user_info.max_budget - user_info.spend
+            ) / user_info.max_budget
         else:
             percent_left = 0
-        verbose_proxy_logger.debug(
-            f"Budget Alerts: Percent left: {percent_left} for {user_info_str}"
-        )
 
         # check if crossed budget
-        if user_current_spend >= user_max_budget:
-            verbose_proxy_logger.debug("Budget Crossed for %s", user_info_str)
-            message = "Budget Crossed for" + user_info_str
-            _cache_key = "budget_alerts:budget_crossed:{}".format(_id)
+        if user_info.spend >= user_info.max_budget:
+            event = "budget_crossed"
+            event_message += "Budget Crossed"
+        elif percent_left <= 0.05:
+            event = "threshold_crossed"
+            event_message += "5% Threshold Crossed"
+        elif percent_left <= 0.15:
+            event = "threshold_crossed"
+            event_message += "15% Threshold Crossed"
+
+        if event is not None and event_group is not None:
+            _cache_key = "budget_alerts:{}:{}".format(event, _id)
             result = await _cache.async_get_cache(key=_cache_key)
             if result is None:
+                webhook_event = WebhookEvent(
+                    event=event,
+                    event_group=event_group,
+                    event_message=event_message,
+                    **user_info_json,
+                )
                 await self.send_alert(
-                    message=message,
+                    message=event_message + "\n\n" + user_info_str,
                     level="High",
                     alert_type="budget_alerts",
-                    user_info=user_info,
+                    user_info=webhook_event,
                 )
                 await _cache.async_set_cache(
                     key=_cache_key,
                     value="SENT",
                     ttl=self.alerting_args.budget_alert_ttl,
                 )
-            return
 
-        # check if 5% of max budget is left
-        if percent_left <= 0.05:
-            message = "5% budget left for" + user_info_str
-            _cache_key = "budget_alerts:5_perc_budget_crossed:{}".format(_id)
-            result = await _cache.async_get_cache(key=_cache_key)
-            if result is None:
-                await self.send_alert(
-                    message=message,
-                    level="Medium",
-                    alert_type="budget_alerts",
-                    user_info=user_info,
-                )
-
-                await _cache.async_set_cache(
-                    key=_cache_key,
-                    value="SENT",
-                    ttl=self.alerting_args.budget_alert_ttl,
-                )
-
-            return
-
-        # check if 15% of max budget is left
-        if percent_left <= 0.15:
-            message = "15% budget left for" + user_info_str
-            _cache_key = "budget_alerts:15_perc_budget_crossed:{}".format(_id)
-            result = await _cache.async_get_cache(key=_cache_key)
-            if result is None:
-                await self.send_alert(
-                    message=message,
-                    level="Low",
-                    alert_type="budget_alerts",
-                    user_info=user_info,
-                )
-                await _cache.async_set_cache(
-                    key=_cache_key,
-                    value="SENT",
-                    ttl=self.alerting_args.budget_alert_ttl,
-                )
             return
         return
 
@@ -785,9 +748,7 @@ Model Info:
     async def model_removed_alert(self, model_name: str):
         pass
 
-    async def send_webhook_alert(
-        self, alert_type: Literal["budget_alerts"], call_info: CallInfo
-    ) -> bool:
+    async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool:
         """
         Sends structured alert to webhook, if set.
 
@@ -800,7 +761,7 @@ Model Info:
         if webhook_url is None:
             raise Exception("Missing webhook_url from environment")
 
-        payload = call_info.model_dump_json()
+        payload = webhook_event.model_dump_json()
         headers = {"Content-type": "application/json"}
 
         response = await self.async_http_handler.post(
@@ -830,7 +791,7 @@ Model Info:
             "new_model_added",
             "cooldown_deployment",
         ],
-        user_info: Optional[CallInfo] = None,
+        user_info: Optional[WebhookEvent] = None,
         **kwargs,
     ):
         """
@@ -855,9 +816,7 @@ Model Info:
             and alert_type == "budget_alerts"
             and user_info is not None
         ):
-            await self.send_webhook_alert(
-                alert_type="budget_alerts", call_info=user_info
-            )
+            await self.send_webhook_alert(webhook_event=user_info)
 
         if "slack" not in self.alerting:
             return
diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml
index 4277fbacb98..a5d91c5a130 100644
--- a/litellm/proxy/_super_secret_config.yaml
+++ b/litellm/proxy/_super_secret_config.yaml
@@ -23,4 +23,4 @@ general_settings:
   alerting: ["slack", "webhook"]
 
 environment_variables: 
-  WEBHOOK_URL: http://0.0.0.0:8000
+  WEBHOOK_URL: https://webhook.site/6ab090e8-c55f-4a23-b075-3209f5c57906
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 443e603c25a..fc7a3de7960 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -1031,11 +1031,12 @@ class TokenCountResponse(LiteLLMBase):
 class CallInfo(LiteLLMBase):
     """Used for slack budget alerting"""
 
+    spend: float
+    max_budget: float
     token: str = Field(description="Hashed value of that key")
-    spend: float = Field(description="The spend for that key")
-    max_budget: Optional[float] = None
     user_id: Optional[str] = None
+    team_id: Optional[str] = None
     user_email: Optional[str] = None
     key_alias: Optional[str] = None
-    projected_exceeded_data: Optional[str] = None
+    projected_exceeded_date: Optional[str] = None
     projected_spend: Optional[float] = None
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index a36e2916341..f2ed8dbff4c 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -597,9 +597,7 @@ async def user_api_key_auth(
                         )
                         asyncio.create_task(
                             proxy_logging_obj.budget_alerts(
-                                user_max_budget=litellm.max_budget,
-                                user_current_spend=global_proxy_spend,
-                                type="user_and_proxy_budget",
+                                type="proxy_budget",
                                 user_info=user_info,
                             )
                         )
@@ -923,17 +921,16 @@ async def user_api_key_auth(
                                 and user_current_spend is not None
                             ):
                                 call_info = CallInfo(
-                                    token=valid_token["token"],
-                                    spend=valid_token["spend"],
+                                    token=valid_token.token,
+                                    spend=user_current_spend,
+                                    max_budget=user_max_budget,
                                     user_id=_user.get("user_id", None),
                                     user_email=_user.get("user_email", None),
-                                    key_alias=valid_token["key_alias"],
+                                    key_alias=valid_token.key_alias,
                                 )
                                 asyncio.create_task(
                                     proxy_logging_obj.budget_alerts(
-                                        user_max_budget=user_max_budget,
-                                        user_current_spend=user_current_spend,
-                                        type="user_and_proxy_budget",
+                                        type="user_budget",
                                         user_info=call_info,
                                     )
                                 )
@@ -955,19 +952,17 @@ async def user_api_key_auth(
                             and user_current_spend is not None
                         ):
                             call_info = CallInfo(
-                                token=valid_token["token"],
-                                spend=valid_token["spend"],
-                                max_budget=valid_token["max_budget"],
+                                token=valid_token.token,
+                                spend=user_current_spend,
+                                max_budget=user_max_budget,
                                 user_id=getattr(user_id_information, "user_id", None),
                                 user_email=getattr(
                                     user_id_information, "user_email", None
                                 ),
-                                key_alias=valid_token["key_alias"],
+                                key_alias=valid_token.key_alias,
                             )
                             asyncio.create_task(
                                 proxy_logging_obj.budget_alerts(
-                                    user_max_budget=user_max_budget,
-                                    user_current_spend=user_current_spend,
                                     type="user_budget",
                                     user_info=call_info,
                                 )
@@ -1000,15 +995,14 @@ async def user_api_key_auth(
             # Check 4. Token Spend is under budget
             if valid_token.spend is not None and valid_token.max_budget is not None:
                 call_info = CallInfo(
-                    token=valid_token["token"],
-                    spend=valid_token["spend"],
-                    max_budget=valid_token["max_budget"],
-                    user_id=valid_token["user_id"],
+                    token=valid_token.token,
+                    spend=valid_token.spend,
+                    max_budget=valid_token.max_budget,
+                    user_id=valid_token.user_id,
+                    team_id=valid_token.team_id,
                 )
                 asyncio.create_task(
                     proxy_logging_obj.budget_alerts(
-                        user_max_budget=valid_token.max_budget,
-                        user_current_spend=valid_token.spend,
                         type="token_budget",
                         user_info=call_info,
                     )
@@ -1055,33 +1049,8 @@ async def user_api_key_auth(
                         raise Exception(
                             f"ExceededModelBudget: Current spend for model: {current_model_spend}; Max Budget for Model: {current_model_budget}"
                         )
-            # Check 6. Token spend is under Team budget
-            if (
-                valid_token.spend is not None
-                and hasattr(valid_token, "team_max_budget")
-                and valid_token.team_max_budget is not None
-            ):
-                call_info = CallInfo(
-                    token=valid_token["token"],
-                    spend=valid_token["spend"],
-                    max_budget=valid_token["max_budget"],
-                    user_id=valid_token["user_id"],
-                )
-                asyncio.create_task(
-                    proxy_logging_obj.budget_alerts(
-                        user_max_budget=valid_token.team_max_budget,
-                        user_current_spend=valid_token.spend,
-                        type="team_budget",
-                        user_info=call_info,
-                    )
-                )
 
-                if valid_token.spend >= valid_token.team_max_budget:
-                    raise Exception(
-                        f"ExceededTokenBudget: Current spend for token: {valid_token.spend}; Max Budget for Team: {valid_token.team_max_budget}"
-                    )
-
-            # Check 7. Team spend is under Team budget
+            # Check 6. Team spend is under Team budget
             if (
                 hasattr(valid_token, "team_spend")
                 and valid_token.team_spend is not None
@@ -1089,15 +1058,13 @@ async def user_api_key_auth(
                 and valid_token.team_max_budget is not None
             ):
                 call_info = CallInfo(
-                    token=valid_token["token"],
-                    spend=valid_token["spend"],
-                    max_budget=valid_token["max_budget"],
-                    user_id=valid_token["user_id"],
+                    token=valid_token.token,
+                    spend=valid_token.team_spend,
+                    max_budget=valid_token.team_max_budget,
+                    user_id=valid_token.user_id,
                 )
                 asyncio.create_task(
                     proxy_logging_obj.budget_alerts(
-                        user_max_budget=valid_token.team_max_budget,
-                        user_current_spend=valid_token.team_spend,
                         type="team_budget",
                         user_info=call_info,
                     )
@@ -1150,12 +1117,11 @@ async def user_api_key_auth(
                         spend=global_proxy_spend,
                         max_budget=litellm.max_budget,
                         user_id=litellm_proxy_admin_name,
+                        team_id=valid_token.team_id,
                     )
                     asyncio.create_task(
                         proxy_logging_obj.budget_alerts(
-                            user_max_budget=litellm.max_budget,
-                            user_current_spend=global_proxy_spend,
-                            type="user_and_proxy_budget",
+                            type="proxy_budget",
                             user_info=call_info,
                         )
                     )
@@ -1344,7 +1310,7 @@ async def user_api_key_auth(
         elif isinstance(e, ProxyException):
             raise e
         raise ProxyException(
-            message="Authentication Error, " + str(e),
+            message="Authentication Error 1234, " + str(e),
             type="auth_error",
             param=getattr(e, "param", "None"),
             code=status.HTTP_401_UNAUTHORIZED,
@@ -1761,14 +1727,14 @@ async def update_cache(
     """
 
     ### UPDATE KEY SPEND ###
-    async def _update_key_cache():
+    async def _update_key_cache(token: str, response_cost: float):
         # Fetch the existing cost for the given token
         if isinstance(token, str) and token.startswith("sk-"):
             hashed_token = hash_token(token=token)
         else:
             hashed_token = token
         verbose_proxy_logger.debug("_update_key_cache: hashed_token=%s", hashed_token)
-        existing_spend_obj = await user_api_key_cache.async_get_cache(key=hashed_token)
+        existing_spend_obj: LiteLLM_VerificationTokenView = await user_api_key_cache.async_get_cache(key=hashed_token)  # type: ignore
         verbose_proxy_logger.debug(
             f"_update_key_cache: existing_spend_obj={existing_spend_obj}"
         )
@@ -1777,7 +1743,7 @@ async def update_cache(
         )
         if existing_spend_obj is None:
             existing_spend = 0
-            existing_spend_obj = LiteLLM_VerificationTokenView()
+            existing_spend_obj = LiteLLM_VerificationTokenView(token=token)
         else:
             existing_spend = existing_spend_obj.spend
         # Calculate the new cost by adding the existing cost and response_cost
@@ -1791,7 +1757,9 @@ async def update_cache(
             and (
                 _is_projected_spend_over_limit(
                     current_spend=new_spend,
-                    soft_budget_limit=existing_spend_obj.litellm_budget_table.soft_budget,
+                    soft_budget_limit=existing_spend_obj.litellm_budget_table[
+                        "soft_budget"
+                    ],
                 )
                 == True
             )
@@ -1807,20 +1775,18 @@ async def update_cache(
             )
             call_info = CallInfo(
                 token=existing_spend_obj.token or "",
-                spend=existing_spend_obj.spend,
+                spend=new_spend,
                 key_alias=existing_spend_obj.key_alias,
-                max_budget=existing_spend_obj.max_budget,
+                max_budget=soft_limit,
                 user_id=existing_spend_obj.user_id,
                 projected_spend=projected_spend,
-                projected_exceeded_data=projected_exceeded_date,
+                projected_exceeded_date=projected_exceeded_date,
             )
             # alert user
             asyncio.create_task(
                 proxy_logging_obj.budget_alerts(
                     type="projected_limit_exceeded",
                     user_info=call_info,
-                    user_max_budget=soft_limit,
-                    user_current_spend=new_spend,
                 )
             )
             # set cooldown on alert
@@ -1830,7 +1796,7 @@ async def update_cache(
             existing_spend_obj is not None
             and getattr(existing_spend_obj, "team_spend", None) is not None
         ):
-            existing_team_spend = existing_spend_obj.team_spend
+            existing_team_spend = existing_spend_obj.team_spend or 0
             # Calculate the new cost by adding the existing cost and response_cost
             existing_spend_obj.team_spend = existing_team_spend + response_cost
 
@@ -1947,8 +1913,8 @@ async def update_cache(
                 f"An error occurred updating end user cache: {str(e)}\n\n{traceback.format_exc()}"
             )
 
-    if token is not None:
-        asyncio.create_task(_update_key_cache())
+    if token is not None and response_cost is not None:
+        asyncio.create_task(_update_key_cache(token=token, response_cost=response_cost))
 
     asyncio.create_task(_update_user_cache())
 
@@ -10231,7 +10197,7 @@ async def test_endpoint(request: Request):
 async def health_services_endpoint(
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
     service: Literal[
-        "slack_budget_alerts", "langfuse", "slack", "openmeter"
+        "slack_budget_alerts", "langfuse", "slack", "openmeter", "webhook"
     ] = fastapi.Query(description="Specify the service being hit."),
 ):
     """
@@ -10246,7 +10212,13 @@ async def health_services_endpoint(
             raise HTTPException(
                 status_code=400, detail={"error": "Service must be specified."}
             )
-        if service not in ["slack_budget_alerts", "langfuse", "slack", "openmeter"]:
+        if service not in [
+            "slack_budget_alerts",
+            "langfuse",
+            "slack",
+            "openmeter",
+            "webhook",
+        ]:
             raise HTTPException(
                 status_code=400,
                 detail={
@@ -10282,6 +10254,20 @@ async def health_services_endpoint(
                 "message": "Mock LLM request made - check langfuse.",
             }
 
+        if service == "webhook":
+            user_info = CallInfo(
+                token=user_api_key_dict.token or "",
+                spend=1,
+                max_budget=0,
+                user_id=user_api_key_dict.user_id,
+                key_alias=user_api_key_dict.key_alias,
+                team_id=user_api_key_dict.team_id,
+            )
+            await proxy_logging_obj.budget_alerts(
+                type="user_budget",
+                user_info=user_info,
+            )
+
         if service == "slack" or service == "slack_budget_alerts":
             if "slack" in general_settings.get("alerting", []):
                 # test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700"""
@@ -10357,6 +10343,7 @@ async def health_services_endpoint(
                     },
                 )
     except Exception as e:
+        traceback.print_exc()
         if isinstance(e, HTTPException):
             raise ProxyException(
                 message=getattr(e, "detail", f"Authentication Error({str(e)})"),
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 4bd793a496c..aab43b1e6e6 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -290,12 +290,9 @@ class ProxyLogging:
             "token_budget",
             "user_budget",
             "team_budget",
-            "user_and_proxy_budget",
-            "failed_budgets",
+            "proxy_budget",
             "projected_limit_exceeded",
         ],
-        user_max_budget: float,
-        user_current_spend: float,
         user_info: CallInfo,
     ):
         if self.alerting is None:
@@ -303,8 +300,6 @@ class ProxyLogging:
             return
         await self.slack_alerting_instance.budget_alerts(
             type=type,
-            user_max_budget=user_max_budget,
-            user_current_spend=user_current_spend,
             user_info=user_info,
         )
 
diff --git a/litellm/tests/test_alerting.py b/litellm/tests/test_alerting.py
index b1887f73d27..20c27082fc1 100644
--- a/litellm/tests/test_alerting.py
+++ b/litellm/tests/test_alerting.py
@@ -431,7 +431,6 @@ async def test_send_daily_reports_all_zero_or_none():
         "user_budget",
         "team_budget",
         "user_and_proxy_budget",
-        "failed_budgets",
         "projected_limit_exceeded",
     ],
 )
@@ -470,7 +469,6 @@ async def test_send_token_budget_crossed_alerts(alerting_type):
         "user_budget",
         "team_budget",
         "user_and_proxy_budget",
-        "failed_budgets",
         "projected_limit_exceeded",
     ],
 )

From 9804eabcd5a56bbd706cd550b9dbd4525cb7d6d0 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 20 May 2024 17:01:34 -0700
Subject: [PATCH 4/6] test(test_alerting.py): fix testing

---
 litellm/tests/test_alerting.py | 34 +++++++++++++++-------------------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/litellm/tests/test_alerting.py b/litellm/tests/test_alerting.py
index 20c27082fc1..2f8d7f3efda 100644
--- a/litellm/tests/test_alerting.py
+++ b/litellm/tests/test_alerting.py
@@ -165,9 +165,9 @@ async def test_budget_alerts_crossed(slack_alerting):
     with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
         await slack_alerting.budget_alerts(
             "user_budget",
-            user_max_budget,
-            user_current_spend,
-            user_info=CallInfo(token="", spend=0),
+            user_info=CallInfo(
+                token="", spend=user_current_spend, max_budget=user_max_budget
+            ),
         )
         mock_send_alert.assert_awaited_once()
 
@@ -180,17 +180,17 @@ async def test_budget_alerts_crossed_again(slack_alerting):
     with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
         await slack_alerting.budget_alerts(
             "user_budget",
-            user_max_budget,
-            user_current_spend,
-            user_info=CallInfo(token="", spend=0),
+            user_info=CallInfo(
+                token="", spend=user_current_spend, max_budget=user_max_budget
+            ),
         )
         mock_send_alert.assert_awaited_once()
         mock_send_alert.reset_mock()
         await slack_alerting.budget_alerts(
             "user_budget",
-            user_max_budget,
-            user_current_spend,
-            user_info=CallInfo(token="", spend=0),
+            user_info=CallInfo(
+                token="", spend=user_current_spend, max_budget=user_max_budget
+            ),
         )
         mock_send_alert.assert_not_awaited()
 
@@ -430,7 +430,7 @@ async def test_send_daily_reports_all_zero_or_none():
         "token_budget",
         "user_budget",
         "team_budget",
-        "user_and_proxy_budget",
+        "proxy_budget",
         "projected_limit_exceeded",
     ],
 )
@@ -441,8 +441,8 @@ async def test_send_token_budget_crossed_alerts(alerting_type):
     with patch.object(slack_alerting, "send_alert", new=AsyncMock()) as mock_send_alert:
         user_info = {
             "token": "50e55ca5bfbd0759697538e8d23c0cd5031f52d9e19e176d7233b20c7c4d3403",
-            "spend": random.uniform(1.5, 1.9),
-            "max_budget": None,
+            "spend": 86,
+            "max_budget": 100,
             "user_id": "ishaan@berri.ai",
             "user_email": "ishaan@berri.ai",
             "key_alias": "my-test-key",
@@ -456,8 +456,6 @@ async def test_send_token_budget_crossed_alerts(alerting_type):
             await slack_alerting.budget_alerts(
                 type=alerting_type,
                 user_info=user_info,
-                user_current_spend=86,
-                user_max_budget=100,
             )
         mock_send_alert.assert_awaited_once()
 
@@ -468,7 +466,7 @@ async def test_send_token_budget_crossed_alerts(alerting_type):
         "token_budget",
         "user_budget",
         "team_budget",
-        "user_and_proxy_budget",
+        "proxy_budget",
         "projected_limit_exceeded",
     ],
 )
@@ -481,8 +479,8 @@ async def test_webhook_alerting(alerting_type):
     ) as mock_send_alert:
         user_info = {
             "token": "50e55ca5bfbd0759697538e8d23c0cd5031f52d9e19e176d7233b20c7c4d3403",
-            "spend": random.uniform(1.5, 1.9),
-            "max_budget": None,
+            "spend": 1,
+            "max_budget": 0,
             "user_id": "ishaan@berri.ai",
             "user_email": "ishaan@berri.ai",
             "key_alias": "my-test-key",
@@ -495,7 +493,5 @@ async def test_webhook_alerting(alerting_type):
             await slack_alerting.budget_alerts(
                 type=alerting_type,
                 user_info=user_info,
-                user_current_spend=86,
-                user_max_budget=100,
             )
         mock_send_alert.assert_awaited_once()

From 72be04a4dec307a4348519b7b3ccb20c98f0b55c Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 20 May 2024 17:26:04 -0700
Subject: [PATCH 5/6] docs(alerting.md): add webhook event spec to docs

---
 docs/my-website/docs/proxy/alerting.md | 84 ++++++++++++++++++++++++--
 1 file changed, 80 insertions(+), 4 deletions(-)

diff --git a/docs/my-website/docs/proxy/alerting.md b/docs/my-website/docs/proxy/alerting.md
index fb49a890134..4f47a77cb3d 100644
--- a/docs/my-website/docs/proxy/alerting.md
+++ b/docs/my-website/docs/proxy/alerting.md
@@ -1,4 +1,4 @@
-# 🚨 Alerting 
+# 🚨 Alerting / Webhooks
 
 Get alerts for:
 
@@ -61,8 +61,7 @@ curl -X GET 'http://localhost:4000/health/services?service=slack' \
   -H 'Authorization: Bearer sk-1234'
 ```
 
-## Advanced
-### Opting into specific alert types
+## Advanced - Opting into specific alert types
 
 Set `alert_types` if you want to Opt into only specific alert types
 
@@ -93,7 +92,7 @@ List[
 ```
 
 
-### Using Discord Webhooks
+## Advanced - Using Discord Webhooks
 
 Discord provides a slack compatible webhook url that you can use for alerting
 
@@ -125,3 +124,80 @@ environment_variables:
 ```
 
 That's it ! You're ready to go !
+
+## Advanced - [BETA] Webhooks for Budget Alerts
+
+**Note**: This is a beta feature, so the spec might change.
+
+Set a webhook to get notified for budget alerts. 
+
+1. Setup config.yaml
+
+Add url to your environment, for testing you can use a link from [here](https://webhook.site/)
+
+```bash
+export WEBHOOK_URL="https://webhook.site/6ab090e8-c55f-4a23-b075-3209f5c57906"
+```
+
+Add 'webhook' to config.yaml
+```yaml
+general_settings: 
+  alerting: ["webhook"] # 👈 KEY CHANGE
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it!
+
+```bash
+curl -X GET --location 'http://0.0.0.0:4000/health/services?service=webhook' \
+--header 'Authorization: Bearer sk-1234'
+```
+
+**Expected Response**
+
+```bash
+{
+  "spend": 1, # the spend for the 'event_group'
+  "max_budget": 0, # the 'max_budget' set for the 'event_group'
+  "token": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
+  "user_id": "default_user_id",
+  "team_id": null,
+  "user_email": null,
+  "key_alias": null,
+  "projected_exceeded_data": null,
+  "projected_spend": null,
+  "event": "budget_crossed", # Literal["budget_crossed", "threshold_crossed", "projected_limit_exceeded"]
+  "event_group": "user",
+  "event_message": "User Budget: Budget Crossed"
+}
+```
+
+**API Spec for Webhook Event**
+
+- `spend` *float*: The current spend amount for the 'event_group'.
+- `max_budget` *float*: The maximum allowed budget for the 'event_group'.
+- `token` *str*: A hashed value of the key, used for authentication or identification purposes.
+- `user_id` *str or null*: The ID of the user associated with the event (optional).
+- `team_id` *str or null*: The ID of the team associated with the event (optional).
+- `user_email` *str or null*: The email of the user associated with the event (optional).
+- `key_alias` *str or null*: An alias for the key associated with the event (optional).
+- `projected_exceeded_date` *str or null*: The date when the budget is projected to be exceeded, returned when 'soft_budget' is set for key (optional).
+- `projected_spend` *float or null*: The projected spend amount, returned when 'soft_budget' is set for key (optional).
+- `event` *Literal["budget_crossed", "threshold_crossed", "projected_limit_exceeded"]*: The type of event that triggered the webhook. Possible values are:
+    * "budget_crossed": Indicates that the spend has exceeded the max budget.
+    * "threshold_crossed": Indicates that spend has crossed a threshold (currently sent when 85% and 95% of budget is reached).
+    * "projected_limit_exceeded": For "key" only - Indicates that the projected spend is expected to exceed the soft budget threshold.
+- `event_group` *Literal["user", "key", "team", "proxy"]*: The group associated with the event. Possible values are:
+    * "user": The event is related to a specific user.
+    * "key": The event is related to a specific key.
+    * "team": The event is related to a team.
+    * "proxy": The event is related to a proxy.
+
+- `event_message` *str*: A human-readable description of the event.
\ No newline at end of file

From 9d815be0b5fd7efc5c7b6a334eb68f499a7efb20 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Mon, 20 May 2024 17:37:41 -0700
Subject: [PATCH 6/6] fix(proxy_server.py): fix error string

---
 litellm/proxy/proxy_server.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index f2ed8dbff4c..42ae1d40969 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -1113,7 +1113,7 @@ async def user_api_key_auth(
 
                 if global_proxy_spend is not None:
                     call_info = CallInfo(
-                        token=valid_token["token"],
+                        token=valid_token.token,
                         spend=global_proxy_spend,
                         max_budget=litellm.max_budget,
                         user_id=litellm_proxy_admin_name,
@@ -1310,7 +1310,7 @@ async def user_api_key_auth(
         elif isinstance(e, ProxyException):
             raise e
         raise ProxyException(
-            message="Authentication Error 1234, " + str(e),
+            message="Authentication Error, " + str(e),
             type="auth_error",
             param=getattr(e, "param", "None"),
             code=status.HTTP_401_UNAUTHORIZED,