From d6f7fa7f4e4789d524d700f24ceb6324e7febda7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 11:42:17 -0700 Subject: [PATCH 1/9] v0 prisma schema --- schema.prisma | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/schema.prisma b/schema.prisma index 5ec73c9dc16..cf979fe8657 100644 --- a/schema.prisma +++ b/schema.prisma @@ -183,6 +183,16 @@ model LiteLLM_SpendLogs { end_user String? } +// View spend, model, api_key per request +model LiteLLM_ErrorLogs { + request_id String @id @default(uuid()) + model_name String @default("") // public model_name / model_group + model_id String @default("") // ID of model in ProxyModelTable + request_kwargs Json @default("{}") + exceptionType String @default("") + exceptionString String @default("") +} + // Beta - allow team members to request access to a model model LiteLLM_UserNotifications { request_id String @id From ac1cabe96345ba59e63ac939e497a48ac56c0ece Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 12:16:03 -0700 Subject: [PATCH 2/9] add LiteLLM_ErrorLogs to types --- litellm/proxy/_types.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index fbe914a26f0..5ec19033acf 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -912,5 +912,15 @@ class LiteLLM_SpendLogs(LiteLLMBase): request_tags: Optional[Json] = None +class LiteLLM_ErrorLogs(LiteLLMBase): + request_id: Optional[str] = str(uuid.uuid4()) + model_name: Optional[str] = "" + model_id: Optional[str] = "" + request_kwargs: Optional[dict] = {} + exception_type: Optional[str] = "" + status_code: Optional[str] = "" + exception_string: Optional[str] = "" + + class LiteLLM_SpendLogs_ResponseObject(LiteLLMBase): response: Optional[List[Union[LiteLLM_SpendLogs, Any]]] = None From c7f979e0fe3c11f5595dca0c18ad7395b728bc3c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 12:31:19 -0700 Subject: [PATCH 3/9] fix schema error logs --- schema.prisma | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/schema.prisma b/schema.prisma index cf979fe8657..7f81ed16a00 100644 --- a/schema.prisma +++ b/schema.prisma @@ -189,8 +189,9 @@ model LiteLLM_ErrorLogs { model_name String @default("") // public model_name / model_group model_id String @default("") // ID of model in ProxyModelTable request_kwargs Json @default("{}") - exceptionType String @default("") - exceptionString String @default("") + exception_type String @default("") + exception_string String @default("") + status_code String @default("") } // Beta - allow team members to request access to a model From 22725bd44df06d97c35d0263fa773cb9eab069ce Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 12:31:33 -0700 Subject: [PATCH 4/9] fix types for errorLog --- litellm/proxy/_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5ec19033acf..9c13b18540e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -916,7 +916,7 @@ class LiteLLM_ErrorLogs(LiteLLMBase): request_id: Optional[str] = str(uuid.uuid4()) model_name: Optional[str] = "" model_id: Optional[str] = "" - request_kwargs: Optional[dict] = {} + request_kwargs: Optional[Json] = {} exception_type: Optional[str] = "" status_code: Optional[str] = "" exception_string: Optional[str] = "" From 06804bc70adf1ac723b776f82ee7830434d45d17 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 12:48:17 -0700 Subject: [PATCH 5/9] fix - working exception writing --- litellm/proxy/proxy_server.py | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 29f3c41dba9..81ea8961a77 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1217,6 +1217,59 @@ def cost_tracking(): litellm.success_callback.append(_PROXY_track_cost_callback) # type: ignore +async def _PROXY_failure_handler( + kwargs, # kwargs to completion + completion_response: litellm.ModelResponse, # response from completion + start_time=None, + end_time=None, # start/end time for completion +): + global prisma_client + if prisma_client is not None: + verbose_proxy_logger.debug( + "inside _PROXY_failure_handler kwargs=", extra=kwargs + ) + + _exception = kwargs.get("exception") + traceback = kwargs.get("traceback") + _exception_type = _exception.__class__.__name__ + + _model = kwargs.get("model", None) + _status_code = _exception.status_code + + _litellm_params = kwargs.get("litellm_params", {}) or {} + _metadata = _litellm_params.get("metadata", {}) or {} + _model_id = _metadata.get("model_info", {}).get("id", None) + verbose_proxy_logger.debug( + "\nexception_type", + _exception_type, + "\nrequest_model", + _model, + "\nmodel_id", + _model_id, + "\nexception", + _exception, + "\ntraceback", + traceback, + ) + error_log = LiteLLM_ErrorLogs( + model_name=_model, + model_id=_model_id, + exception_type=_exception_type, + status_code=_status_code, + exception_string=str(_exception), + ) + + # helper function to convert to dict on pydantic v2 & v1 + error_log_dict = _get_pydantic_json_dict(error_log) + error_log_dict["request_kwargs"] = json.dumps(error_log_dict["request_kwargs"]) + + await prisma_client.db.litellm_errorlogs.create( + data=error_log_dict # type: ignore + ) + + pass + + async def _PROXY_track_cost_callback( kwargs, # kwargs to completion completion_response: litellm.ModelResponse, # response from completion @@ -1302,6 +1355,15 @@ async def _PROXY_track_cost_callback( verbose_proxy_logger.debug("error in tracking cost callback - %s", e) +def error_tracking(): + global prisma_client, custom_db_client + if prisma_client is not None or custom_db_client is not None: + if isinstance(litellm.failure_callback, list): + verbose_proxy_logger.debug("setting litellm failure callback to track cost") + if (_PROXY_failure_handler) not in litellm.failure_callback: # type: ignore + litellm.failure_callback.append(_PROXY_failure_handler) # type: ignore + + def _set_spend_logs_payload( payload: dict, prisma_client: PrismaClient, spend_logs_url: Optional[str] = None ): @@ -3194,6 +3256,9 @@ async def startup_event(): ## COST TRACKING ## cost_tracking() + ## Error Tracking ## + error_tracking() + db_writer_client = HTTPHandler() proxy_logging_obj._init_litellm_callbacks() # INITIALIZE LITELLM CALLBACKS ON SERVER STARTUP <- do this to catch any logging errors on startup, not when calls are being made From ee2a2ce559bdb22d4480e42be3988a36157c9e5d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 13:02:42 -0700 Subject: [PATCH 6/9] fix - log api_base in errors --- litellm/proxy/_types.py | 1 + litellm/proxy/proxy_server.py | 5 +++++ schema.prisma | 1 + 3 files changed, 7 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9c13b18540e..9548025d51d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -914,6 +914,7 @@ class LiteLLM_SpendLogs(LiteLLMBase): class LiteLLM_ErrorLogs(LiteLLMBase): request_id: Optional[str] = str(uuid.uuid4()) + api_base: Optional[str] = "" model_name: Optional[str] = "" model_id: Optional[str] = "" request_kwargs: Optional[Json] = {} diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 81ea8961a77..c9d94288e1b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1239,6 +1239,9 @@ async def _PROXY_failure_handler( _litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = _litellm_params.get("metadata", {}) or {} _model_id = _metadata.get("model_info", {}).get("id", None) + + api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) + verbose_proxy_logger.debug( "\nexception_type", _exception_type, @@ -1252,8 +1255,10 @@ async def _PROXY_failure_handler( traceback, ) error_log = LiteLLM_ErrorLogs( + request_id=str(uuid.uuid4()), model_name=_model, model_id=_model_id, + api_base=api_base, exception_type=_exception_type, status_code=_status_code, exception_string=str(_exception), diff --git a/schema.prisma b/schema.prisma index 7f81ed16a00..e9acfe18829 100644 --- a/schema.prisma +++ b/schema.prisma @@ -186,6 +186,7 @@ model LiteLLM_SpendLogs { // View spend, model, api_key per request model LiteLLM_ErrorLogs { request_id String @id @default(uuid()) + api_base String @default("") model_name String @default("") // public model_name / model_group model_id String @default("") // ID of model in ProxyModelTable request_kwargs Json @default("{}") From ad5fddef156804bafcced6a9e19c97d3d84faf57 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 13:11:09 -0700 Subject: [PATCH 7/9] fix log model_group --- litellm/proxy/_types.py | 2 +- litellm/proxy/proxy_server.py | 7 ++++--- schema.prisma | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9548025d51d..75dfdb4c4db 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -915,7 +915,7 @@ class LiteLLM_SpendLogs(LiteLLMBase): class LiteLLM_ErrorLogs(LiteLLMBase): request_id: Optional[str] = str(uuid.uuid4()) api_base: Optional[str] = "" - model_name: Optional[str] = "" + model_group: Optional[str] = "" model_id: Optional[str] = "" request_kwargs: Optional[Json] = {} exception_type: Optional[str] = "" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c9d94288e1b..f9e7756bde6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1232,13 +1232,14 @@ async def _PROXY_failure_handler( _exception = kwargs.get("exception") traceback = kwargs.get("traceback") _exception_type = _exception.__class__.__name__ - _model = kwargs.get("model", None) + _status_code = _exception.status_code _litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = _litellm_params.get("metadata", {}) or {} - _model_id = _metadata.get("model_info", {}).get("id", None) + _model_id = _metadata.get("model_info", {}).get("id", "") + _model_group = _metadata.get("model_group", "") api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) @@ -1256,7 +1257,7 @@ async def _PROXY_failure_handler( ) error_log = LiteLLM_ErrorLogs( request_id=str(uuid.uuid4()), - model_name=_model, + model_group=_model_group, model_id=_model_id, api_base=api_base, exception_type=_exception_type, diff --git a/schema.prisma b/schema.prisma index e9acfe18829..cd02199835d 100644 --- a/schema.prisma +++ b/schema.prisma @@ -187,7 +187,7 @@ model LiteLLM_SpendLogs { model LiteLLM_ErrorLogs { request_id String @id @default(uuid()) api_base String @default("") - model_name String @default("") // public model_name / model_group + model_group String @default("") // public model_name / model_group model_id String @default("") // ID of model in ProxyModelTable request_kwargs Json @default("{}") exception_type String @default("") From 3aad034a8b0c52275944ab3ab0276059170716b2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 13:28:26 -0700 Subject: [PATCH 8/9] feat log request kwargs in error logs --- litellm/proxy/_types.py | 2 +- litellm/proxy/proxy_server.py | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 75dfdb4c4db..2b3a72250f1 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -917,7 +917,7 @@ class LiteLLM_ErrorLogs(LiteLLMBase): api_base: Optional[str] = "" model_group: Optional[str] = "" model_id: Optional[str] = "" - request_kwargs: Optional[Json] = {} + request_kwargs: Optional[dict] = {} exception_type: Optional[str] = "" status_code: Optional[str] = "" exception_string: Optional[str] = "" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f9e7756bde6..530438b94a8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1230,39 +1230,39 @@ async def _PROXY_failure_handler( ) _exception = kwargs.get("exception") - traceback = kwargs.get("traceback") _exception_type = _exception.__class__.__name__ _model = kwargs.get("model", None) - _status_code = _exception.status_code + _optional_params = kwargs.get("optional_params", {}) + _optional_params = copy.deepcopy(_optional_params) + + for k, v in _optional_params.items(): + v = str(v) + v = v[:100] + + _status_code = "500" + try: + _status_code = str(_exception.status_code) + except: + # Don't let this fail logging the exception to the dB + pass _litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = _litellm_params.get("metadata", {}) or {} _model_id = _metadata.get("model_info", {}).get("id", "") _model_group = _metadata.get("model_group", "") - api_base = litellm.get_api_base(model=_model, optional_params=_litellm_params) + _exception_string = str(_exception)[:500] - verbose_proxy_logger.debug( - "\nexception_type", - _exception_type, - "\nrequest_model", - _model, - "\nmodel_id", - _model_id, - "\nexception", - _exception, - "\ntraceback", - traceback, - ) error_log = LiteLLM_ErrorLogs( request_id=str(uuid.uuid4()), model_group=_model_group, model_id=_model_id, + request_kwargs=_optional_params, api_base=api_base, exception_type=_exception_type, status_code=_status_code, - exception_string=str(_exception), + exception_string=_exception_string, ) # helper function to convert to dict on pydantic v2 & v1 From 4b8fda4ac408825cc0c80e1f81212c774aa6af87 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 13:34:14 -0700 Subject: [PATCH 9/9] log startTime and EndTime for exceptions --- litellm/proxy/_types.py | 2 ++ litellm/proxy/proxy_server.py | 2 ++ schema.prisma | 2 ++ 3 files changed, 6 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 2b3a72250f1..c910664f15e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -921,6 +921,8 @@ class LiteLLM_ErrorLogs(LiteLLMBase): exception_type: Optional[str] = "" status_code: Optional[str] = "" exception_string: Optional[str] = "" + startTime: Union[str, datetime, None] + endTime: Union[str, datetime, None] class LiteLLM_SpendLogs_ResponseObject(LiteLLMBase): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 530438b94a8..3a7821d2727 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1263,6 +1263,8 @@ async def _PROXY_failure_handler( exception_type=_exception_type, status_code=_status_code, exception_string=_exception_string, + startTime=kwargs.get("start_time"), + endTime=kwargs.get("end_time"), ) # helper function to convert to dict on pydantic v2 & v1 diff --git a/schema.prisma b/schema.prisma index cd02199835d..b362a0ec025 100644 --- a/schema.prisma +++ b/schema.prisma @@ -186,6 +186,8 @@ model LiteLLM_SpendLogs { // View spend, model, api_key per request model LiteLLM_ErrorLogs { request_id String @id @default(uuid()) + startTime DateTime // Assuming start_time is a DateTime field + endTime DateTime // Assuming end_time is a DateTime field api_base String @default("") model_group String @default("") // public model_name / model_group model_id String @default("") // ID of model in ProxyModelTable