From 30a54ef00728d087db41b3a464bf38a43e56444a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 12 Aug 2024 10:56:11 -0700 Subject: [PATCH 1/7] v0 log KeyCreatedEvent --- litellm/integrations/slack_alerting.py | 49 +++++++++++++++++++++++++- litellm/proxy/_types.py | 9 +++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index bb6698d7308..0ad403a9790 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -26,7 +26,13 @@ from litellm.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.proxy._types import AlertType, CallInfo, UserAPIKeyAuth, WebhookEvent +from litellm.proxy._types import ( + AlertType, + CallInfo, + KeyCreatedEvent, + UserAPIKeyAuth, + WebhookEvent, +) from litellm.types.router import LiteLLM_Params from .email_templates.templates import * @@ -1788,3 +1794,44 @@ Model Info: verbose_proxy_logger.error("Error sending weekly spend report %s", e) pass + + async def send_virtual_key_event_slack( + self, + key_event: KeyCreatedEvent, + event_name: str, + ): + """ + Helper to send fallback statistics from prometheus server -> to slack + + This runs once per day and sends an overview of all the fallback statistics + """ + try: + message = f"`{event_name}`\n" + + key_event_dict = key_event.model_dump() + + # Add Created by information first + message += "*Created by:*\n" + for key, value in key_event_dict.items(): + if "created_by" in key: + message += f"{key}: `{value}`\n" + + # Add all non created by information next + message += "\n*Arguments passed:*\n" + for key, value in key_event_dict.items(): + if "created_by" not in key: + message += f"{key}: `{value}`\n" + + await self.send_alert( + message=message, + level="High", + alert_type="fallback_reports", + alerting_metadata={}, + ) + + except Exception as e: + verbose_proxy_logger.error( + "Error sending send_virtual_key_event_slack %s", e + ) + + return diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5e0de6fb09a..0533b39628e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1754,3 +1754,12 @@ class SSOUserDefinedValues(TypedDict): user_role: Optional[str] max_budget: Optional[float] budget_duration: Optional[str] + + +class KeyCreatedEvent(LiteLLMBase): + created_by_user_id: str + created_by_user_role: str + created_by_key_alias: Optional[str] + key_alias: Optional[str] + team_id: Optional[str] + max_budget: Optional[float] From 47953aaa96bf979e3e8349d06d3a40d8e4de53d8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 12 Aug 2024 11:02:21 -0700 Subject: [PATCH 2/7] use management_endpoint_wrapper for key endpoints --- .../proxy/management_endpoints/key_management_endpoints.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 0e4696e4423..968b3ede912 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -26,6 +26,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.proxy.utils import _duration_in_seconds router = APIRouter() @@ -37,6 +38,7 @@ router = APIRouter() dependencies=[Depends(user_api_key_auth)], response_model=GenerateKeyResponse, ) +@management_endpoint_wrapper async def generate_key_fn( data: GenerateKeyRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -278,6 +280,7 @@ async def generate_key_fn( @router.post( "/key/update", tags=["key management"], dependencies=[Depends(user_api_key_auth)] ) +@management_endpoint_wrapper async def update_key_fn( request: Request, data: UpdateKeyRequest, @@ -398,6 +401,7 @@ async def update_key_fn( @router.post( "/key/delete", tags=["key management"], dependencies=[Depends(user_api_key_auth)] ) +@management_endpoint_wrapper async def delete_key_fn( data: KeyRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), From 1335859e5d55ffc119a219ae4b455d68b3eca3f3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 12 Aug 2024 11:07:48 -0700 Subject: [PATCH 3/7] send management endpoint alert --- litellm/proxy/management_helpers/utils.py | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index 86c761a70ce..f37f0b7d243 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -13,6 +13,7 @@ from litellm.proxy._types import ( # key request types; user request types; tea DeleteCustomerRequest, DeleteTeamRequest, DeleteUserRequest, + KeyCreatedEvent, KeyRequest, LiteLLM_TeamTable, ManagementEndpointLoggingPayload, @@ -186,6 +187,40 @@ def _delete_customer_id_from_cache(kwargs): pass +async def send_management_endpoint_alert( + request_kwargs: dict, + user_api_key_dict: UserAPIKeyAuth, + function_name: str, +): + """ + Sends a slack alert when: + - A virtual key is created, updated, or deleted + - An internal user is created, updated, or deleted + - A team is created, updated, or deleted + """ + from litellm.proxy.proxy_server import proxy_logging_obj + + if ( + proxy_logging_obj is not None + and proxy_logging_obj.slack_alerting_instance is not None + ): + key_event = KeyCreatedEvent( + created_by_user_id=user_api_key_dict.user_id or "Unknown", + created_by_user_role=user_api_key_dict.user_role or "Unknown", + created_by_key_alias=user_api_key_dict.key_alias, + key_alias=request_kwargs.get("key_alias"), + team_id=request_kwargs.get("team_id"), + max_budget=request_kwargs.get("max_budget"), + ) + + if function_name == "generate_key_fn": + from litellm.proxy.proxy_server import proxy_logging_obj + + await proxy_logging_obj.slack_alerting_instance.send_virtual_key_event_slack( + key_event=key_event, event_name="Virtual Key Created" + ) + + def management_endpoint_wrapper(func): """ This wrapper does the following: @@ -207,6 +242,13 @@ def management_endpoint_wrapper(func): user_api_key_dict: UserAPIKeyAuth = ( kwargs.get("user_api_key_dict") or UserAPIKeyAuth() ) + + await send_management_endpoint_alert( + request_kwargs=kwargs, + user_api_key_dict=user_api_key_dict, + function_name=func.__name__, + ) + _http_request: Request = kwargs.get("http_request") parent_otel_span = user_api_key_dict.parent_otel_span if parent_otel_span is not None: From 65ac139b6b167ab3f4aa94941d3d56e52494b078 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 12 Aug 2024 11:12:38 -0700 Subject: [PATCH 4/7] make send_management_endpoint_alert a premium feature --- litellm/proxy/management_helpers/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index f37f0b7d243..b5a55927626 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -198,7 +198,10 @@ async def send_management_endpoint_alert( - An internal user is created, updated, or deleted - A team is created, updated, or deleted """ - from litellm.proxy.proxy_server import proxy_logging_obj + from litellm.proxy.proxy_server import premium_user, proxy_logging_obj + + if premium_user is not True: + return if ( proxy_logging_obj is not None From 4435a23121387aa146492c0c8e95503b66764111 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 12 Aug 2024 11:39:24 -0700 Subject: [PATCH 5/7] send alert on all key events --- litellm/integrations/slack_alerting.py | 16 +++++++------ litellm/proxy/_types.py | 6 ++--- litellm/proxy/management_helpers/utils.py | 29 +++++++++++++---------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 0ad403a9790..d0a76ec9635 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -29,8 +29,8 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import ( AlertType, CallInfo, - KeyCreatedEvent, UserAPIKeyAuth, + VirtualKeyEvent, WebhookEvent, ) from litellm.types.router import LiteLLM_Params @@ -1797,7 +1797,7 @@ Model Info: async def send_virtual_key_event_slack( self, - key_event: KeyCreatedEvent, + key_event: VirtualKeyEvent, event_name: str, ): """ @@ -1811,16 +1811,18 @@ Model Info: key_event_dict = key_event.model_dump() # Add Created by information first - message += "*Created by:*\n" + message += "*Action Done by:*\n" for key, value in key_event_dict.items(): if "created_by" in key: message += f"{key}: `{value}`\n" - # Add all non created by information next + # Add args sent to function in the alert message += "\n*Arguments passed:*\n" - for key, value in key_event_dict.items(): - if "created_by" not in key: - message += f"{key}: `{value}`\n" + request_kwargs = key_event.request_kwargs + for key, value in request_kwargs.items(): + if key == "user_api_key_dict": + continue + message += f"{key}: `{value}`\n" await self.send_alert( message=message, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0533b39628e..deb496f2e91 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1756,10 +1756,8 @@ class SSOUserDefinedValues(TypedDict): budget_duration: Optional[str] -class KeyCreatedEvent(LiteLLMBase): +class VirtualKeyEvent(LiteLLMBase): created_by_user_id: str created_by_user_role: str created_by_key_alias: Optional[str] - key_alias: Optional[str] - team_id: Optional[str] - max_budget: Optional[float] + request_kwargs: dict diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index b5a55927626..2de7db35271 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -13,7 +13,6 @@ from litellm.proxy._types import ( # key request types; user request types; tea DeleteCustomerRequest, DeleteTeamRequest, DeleteUserRequest, - KeyCreatedEvent, KeyRequest, LiteLLM_TeamTable, ManagementEndpointLoggingPayload, @@ -24,6 +23,7 @@ from litellm.proxy._types import ( # key request types; user request types; tea UpdateTeamRequest, UpdateUserRequest, UserAPIKeyAuth, + VirtualKeyEvent, ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.utils import PrismaClient @@ -203,24 +203,29 @@ async def send_management_endpoint_alert( if premium_user is not True: return + key_function_to_event_name = { + "generate_key_fn": "Virtual Key Created", + "update_key_fn": "Virtual Key Updated", + "delete_key_fn": "Virtual Key Deleted", + } + if ( proxy_logging_obj is not None and proxy_logging_obj.slack_alerting_instance is not None ): - key_event = KeyCreatedEvent( - created_by_user_id=user_api_key_dict.user_id or "Unknown", - created_by_user_role=user_api_key_dict.user_role or "Unknown", - created_by_key_alias=user_api_key_dict.key_alias, - key_alias=request_kwargs.get("key_alias"), - team_id=request_kwargs.get("team_id"), - max_budget=request_kwargs.get("max_budget"), - ) - if function_name == "generate_key_fn": - from litellm.proxy.proxy_server import proxy_logging_obj + # Virtual Key Events + if function_name in key_function_to_event_name: + key_event = VirtualKeyEvent( + created_by_user_id=user_api_key_dict.user_id or "Unknown", + created_by_user_role=user_api_key_dict.user_role or "Unknown", + created_by_key_alias=user_api_key_dict.key_alias, + request_kwargs=request_kwargs, + ) + event_name = key_function_to_event_name[function_name] await proxy_logging_obj.slack_alerting_instance.send_virtual_key_event_slack( - key_event=key_event, event_name="Virtual Key Created" + key_event=key_event, event_name=event_name ) From 06eb9c4e3948a0d3f53fc6a4503a2124044f2ad2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 12 Aug 2024 11:56:13 -0700 Subject: [PATCH 6/7] send slack alert on team events --- litellm/proxy/management_helpers/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index 2de7db35271..438e2ca1c7d 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -203,10 +203,16 @@ async def send_management_endpoint_alert( if premium_user is not True: return - key_function_to_event_name = { + management_function_to_event_name = { "generate_key_fn": "Virtual Key Created", "update_key_fn": "Virtual Key Updated", "delete_key_fn": "Virtual Key Deleted", + # Team events + "new_team": "New Team Created", + "update_team": "Team Updated", + "delete_team": "Team Deleted", + # Internal User events + "user_update": "Internal User Updated", } if ( @@ -215,7 +221,7 @@ async def send_management_endpoint_alert( ): # Virtual Key Events - if function_name in key_function_to_event_name: + if function_name in management_function_to_event_name: key_event = VirtualKeyEvent( created_by_user_id=user_api_key_dict.user_id or "Unknown", created_by_user_role=user_api_key_dict.user_role or "Unknown", @@ -223,7 +229,7 @@ async def send_management_endpoint_alert( request_kwargs=request_kwargs, ) - event_name = key_function_to_event_name[function_name] + event_name = management_function_to_event_name[function_name] await proxy_logging_obj.slack_alerting_instance.send_virtual_key_event_slack( key_event=key_event, event_name=event_name ) From a44bd23631d3604978654f0b18802636d9734e41 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 12 Aug 2024 12:01:26 -0700 Subject: [PATCH 7/7] log new_user, delete user --- litellm/proxy/management_helpers/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index 438e2ca1c7d..64e691196de 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -204,7 +204,7 @@ async def send_management_endpoint_alert( return management_function_to_event_name = { - "generate_key_fn": "Virtual Key Created", + "generate_key_fn": "New Virtual Key Created", "update_key_fn": "Virtual Key Updated", "delete_key_fn": "Virtual Key Deleted", # Team events @@ -212,7 +212,9 @@ async def send_management_endpoint_alert( "update_team": "Team Updated", "delete_team": "Team Deleted", # Internal User events + "new_user": "New Internal User Created", "user_update": "Internal User Updated", + "delete_user": "Internal User Deleted", } if (