diff --git a/enterprise/dist/litellm_enterprise-0.1.29-py3-none-any.whl b/enterprise/dist/litellm_enterprise-0.1.29-py3-none-any.whl index 0895ecbc427..5eedd1fe876 100644 Binary files a/enterprise/dist/litellm_enterprise-0.1.29-py3-none-any.whl and b/enterprise/dist/litellm_enterprise-0.1.29-py3-none-any.whl differ diff --git a/enterprise/dist/litellm_enterprise-0.1.29.tar.gz b/enterprise/dist/litellm_enterprise-0.1.29.tar.gz index 6781cf26cc9..1ea224f8f34 100644 Binary files a/enterprise/dist/litellm_enterprise-0.1.29.tar.gz and b/enterprise/dist/litellm_enterprise-0.1.29.tar.gz differ diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py index 61e0745bab1..1e87e5594fc 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py @@ -30,8 +30,15 @@ from litellm.integrations.email_templates.user_invitation_email import ( from litellm.integrations.email_templates.templates import ( MAX_BUDGET_ALERT_EMAIL_TEMPLATE, SOFT_BUDGET_ALERT_EMAIL_TEMPLATE, + TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE, +) +from litellm.proxy._types import ( + CallInfo, + InvitationNew, + Litellm_EntityType, + UserAPIKeyAuth, + WebhookEvent, ) -from litellm.proxy._types import CallInfo, InvitationNew, UserAPIKeyAuth, WebhookEvent from litellm.secret_managers.main import get_secret_bool from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL from litellm.constants import ( @@ -217,6 +224,68 @@ class BaseEmailLogger(CustomLogger): ) pass + async def send_team_soft_budget_alert_email(self, event: WebhookEvent): + """ + Send email to team members when team soft budget is crossed + Supports multiple recipients via alert_emails field from team metadata + """ + # Collect all recipient emails + recipient_emails: List[str] = [] + + # Add additional alert emails from team metadata.soft_budget_alert_emails + if hasattr(event, "alert_emails") and event.alert_emails: + for email in event.alert_emails: + if email and email not in recipient_emails: # Avoid duplicates + recipient_emails.append(email) + + # If no recipients found, skip sending + if not recipient_emails: + verbose_proxy_logger.warning( + f"No recipient emails found for team soft budget alert. event={event.model_dump(exclude_none=True)}" + ) + return + + verbose_proxy_logger.debug( + f"send_team_soft_budget_alert_email_event: {json.dumps(event.model_dump(exclude_none=True), indent=4, default=str)}" + ) + + # Get email params using the first recipient email (for template formatting) + email_params = await self._get_email_params( + email_event=EmailEvent.soft_budget_crossed, + user_id=event.user_id, + user_email=recipient_emails[0], + event_message=event.event_message, + ) + + # Format budget values + soft_budget_str = f"${event.soft_budget}" if event.soft_budget is not None else "N/A" + spend_str = f"${event.spend}" if event.spend is not None else "$0.00" + max_budget_info = "" + if event.max_budget is not None: + max_budget_info = f"Maximum Budget: ${event.max_budget}
" + + # Use team alias or generic greeting + team_alias = event.team_alias or "Team" + + email_html_content = TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE.format( + email_logo_url=email_params.logo_url, + team_alias=team_alias, + soft_budget=soft_budget_str, + spend=spend_str, + max_budget_info=max_budget_info, + base_url=email_params.base_url, + email_support_contact=email_params.support_contact, + ) + + # Send email to all recipients + await self.send_email( + from_email=self.DEFAULT_LITELLM_EMAIL, + to_email=recipient_emails, + subject=email_params.subject, + html_body=email_html_content, + ) + pass + async def send_max_budget_alert_email(self, event: WebhookEvent): """ Send email to user when max budget alert threshold is reached @@ -285,15 +354,29 @@ class BaseEmailLogger(CustomLogger): # - Don't re-alert, if alert already sent _cache: DualCache = self.internal_usage_cache - # percent of max_budget left to spend - if user_info.max_budget is None and user_info.soft_budget is None: - return - # For soft_budget alerts, check if we've already sent an alert if type == "soft_budget": + # For team soft budget alerts, we only need team soft_budget to be set + # For other entity types, we need either max_budget or soft_budget + if user_info.event_group == Litellm_EntityType.TEAM: + if user_info.soft_budget is None: + return + else: + # For non-team alerts, require either max_budget or soft_budget + if user_info.max_budget is None and user_info.soft_budget is None: + return if user_info.soft_budget is not None and user_info.spend >= user_info.soft_budget: # Generate cache key based on event type and identifier - _id = user_info.token or user_info.user_id or "default_id" + # Use appropriate ID based on event_group to ensure unique cache keys per entity type + if user_info.event_group == Litellm_EntityType.TEAM: + _id = user_info.team_id or "default_id" + elif user_info.event_group == Litellm_EntityType.ORGANIZATION: + _id = user_info.organization_id or "default_id" + elif user_info.event_group == Litellm_EntityType.USER: + _id = user_info.user_id or "default_id" + else: + # For KEY and other types, use token or user_id + _id = user_info.token or user_info.user_id or "default_id" _cache_key = f"email_budget_alerts:soft_budget_crossed:{_id}" # Check if we've already sent this alert @@ -318,10 +401,15 @@ class BaseEmailLogger(CustomLogger): projected_exceeded_date=user_info.projected_exceeded_date, projected_spend=user_info.projected_spend, event_group=user_info.event_group, + alert_emails=user_info.alert_emails, ) try: - await self.send_soft_budget_alert_email(webhook_event) + # Use team-specific function for team alerts, otherwise use standard function + if user_info.event_group == Litellm_EntityType.TEAM: + await self.send_team_soft_budget_alert_email(webhook_event) + else: + await self.send_soft_budget_alert_email(webhook_event) # Cache the alert to prevent duplicate sends await _cache.async_set_cache( diff --git a/litellm/integrations/email_templates/templates.py b/litellm/integrations/email_templates/templates.py index 5de23db0f24..091351df2bb 100644 --- a/litellm/integrations/email_templates/templates.py +++ b/litellm/integrations/email_templates/templates.py @@ -85,6 +85,30 @@ SOFT_BUDGET_ALERT_EMAIL_TEMPLATE = """ The LiteLLM team
""" +TEAM_SOFT_BUDGET_ALERT_EMAIL_TEMPLATE = """ + LiteLLM Logo + +

Hi {team_alias} team member,
+ + Your LiteLLM team has crossed its soft budget limit of {soft_budget}.

+ + Current Spend: {spend}
+ Soft Budget: {soft_budget}
+ {max_budget_info} + +

+ ⚠️ Note: Your API requests will continue to work, but you should monitor your usage closely. + If you reach your maximum budget, requests will be rejected. +

+ + You can view your usage and manage your budget in the LiteLLM Dashboard.

+ + If you have any questions, please send an email to {email_support_contact}

+ + Best,
+ The LiteLLM team
+""" + MAX_BUDGET_ALERT_EMAIL_TEMPLATE = """ LiteLLM Logo diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5b697249a41..f38f94f4c98 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2190,6 +2190,7 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): team_tpm_limit: Optional[int] = None team_rpm_limit: Optional[int] = None team_max_budget: Optional[float] = None + team_soft_budget: Optional[float] = None team_models: List = [] team_blocked: bool = False soft_budget: Optional[float] = None @@ -2648,6 +2649,10 @@ class CallInfo(LiteLLMPydanticObjectBase): projected_exceeded_date: Optional[str] = None projected_spend: Optional[float] = None event_group: Litellm_EntityType + alert_emails: Optional[List[str]] = Field( + default=None, + description="Additional email addresses to send alerts to (e.g., from team metadata)", + ) class WebhookEvent(CallInfo): diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 359bb944546..437ef0d438b 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -218,6 +218,13 @@ async def common_checks( valid_token=valid_token, ) + # 3.0.5. If team is over soft budget (alert only, doesn't block) + await _team_soft_budget_check( + team_object=team_object, + proxy_logging_obj=proxy_logging_obj, + valid_token=valid_token, + ) + # 3.1. If organization is in budget await _organization_max_budget_check( valid_token=valid_token, @@ -2421,6 +2428,66 @@ async def _team_max_budget_check( ) +async def _team_soft_budget_check( + team_object: Optional[LiteLLM_TeamTable], + valid_token: Optional[UserAPIKeyAuth], + proxy_logging_obj: ProxyLogging, +): + """ + Triggers a budget alert if the team is over it's soft budget. + """ + if ( + team_object is not None + and team_object.soft_budget is not None + and team_object.spend is not None + and team_object.spend >= team_object.soft_budget + ): + verbose_proxy_logger.debug( + "Crossed Soft Budget for team %s, spend %s, soft_budget %s", + team_object.team_id, + team_object.spend, + team_object.soft_budget, + ) + if valid_token: + # Extract alert emails from team metadata + alert_emails: Optional[List[str]] = None + if team_object.metadata is not None and isinstance(team_object.metadata, dict): + soft_budget_alert_emails = team_object.metadata.get("soft_budget_alerting_emails") + if soft_budget_alert_emails is not None: + if isinstance(soft_budget_alert_emails, list): + alert_emails = [email for email in soft_budget_alert_emails if isinstance(email, str) and email.strip()] + elif isinstance(soft_budget_alert_emails, str): + # Handle comma-separated string + alert_emails = [email.strip() for email in soft_budget_alert_emails.split(",") if email.strip()] + # Filter out empty strings + if alert_emails: + alert_emails = [email for email in alert_emails if email] + else: + alert_emails = None + + call_info = CallInfo( + token=valid_token.token, + spend=team_object.spend, + max_budget=team_object.max_budget, + soft_budget=team_object.soft_budget, + user_id=valid_token.user_id, + team_id=valid_token.team_id, + team_alias=valid_token.team_alias, + organization_id=valid_token.org_id, + user_email=None, # Team-level alert, no specific user email + key_alias=valid_token.key_alias, + event_group=Litellm_EntityType.TEAM, + alert_emails=alert_emails, + ) + + asyncio.create_task( + proxy_logging_obj.budget_alerts( + type="soft_budget", + user_info=call_info, + ) + ) + + async def _organization_max_budget_check( valid_token: Optional[UserAPIKeyAuth], team_object: Optional[LiteLLM_TeamTable], diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index a153c6e51cc..05eeab3f611 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -1149,6 +1149,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 _team_obj: Optional[LiteLLM_TeamTable] = LiteLLM_TeamTable( team_id=valid_token.team_id, max_budget=valid_token.team_max_budget, + soft_budget=valid_token.team_soft_budget, spend=valid_token.team_spend, tpm_limit=valid_token.team_tpm_limit, rpm_limit=valid_token.team_rpm_limit, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 6bbf0df74de..00491675876 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1370,17 +1370,35 @@ class ProxyLogging: ], user_info: CallInfo, ): - if self.alerting is None: - # do nothing if alerting is not switched on + # For soft_budget alerts with alert_emails set, allow email sending even if alerting is None + # This enables team-specific soft budget email alerts via metadata.soft_budget_alerting_emails + # Note: user_info is a CallInfo that can represent user/team/org level info. For team budgets, + # alert_emails is populated from team_object.metadata.soft_budget_alerting_emails (see auth_checks.py) + is_soft_budget_with_alert_emails = ( + type == "soft_budget" + and user_info.alert_emails is not None + and len(user_info.alert_emails) > 0 + ) + + if self.alerting is None and not is_soft_budget_with_alert_emails: + # do nothing if alerting is not switched on (unless it's a soft_budget alert with team-specific emails) return - if "slack" in self.alerting: + if self.alerting is not None and "slack" in self.alerting: await self.slack_alerting_instance.budget_alerts( type=type, user_info=user_info, ) - if "email" in self.alerting and self.email_logging_instance is not None: + # Call email_logging_instance if: + # 1. "email" is in alerting config, OR + # 2. It's a soft_budget alert with team-specific alert_emails (bypasses global alerting config) + should_send_email = ( + (self.alerting is not None and "email" in self.alerting) + or is_soft_budget_with_alert_emails + ) + + if should_send_email and self.email_logging_instance is not None: await self.email_logging_instance.budget_alerts( type=type, user_info=user_info, @@ -2607,7 +2625,8 @@ class PrismaClient: SELECT v.*, t.spend AS team_spend, - t.max_budget AS team_max_budget, + t.max_budget AS team_max_budget, + t.soft_budget AS team_soft_budget, t.tpm_limit AS team_tpm_limit, t.rpm_limit AS team_rpm_limit, t.models AS team_models,