From b1b21b03404bc74497b84d7f1e02249a1d1da28a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 5 Jul 2024 14:39:35 -0700 Subject: [PATCH 1/3] fix(proxy/utils.py): support logging rejected requests to langfuse, etc. --- litellm/litellm_core_utils/litellm_logging.py | 5 --- litellm/proxy/utils.py | 40 ++++--------------- litellm/tests/test_proxy_reject_logging.py | 35 +++++++++++----- litellm/utils.py | 8 ++-- 4 files changed, 36 insertions(+), 52 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4edbce5e15b..c0fdd3090c2 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -153,11 +153,6 @@ class Logging: langfuse_secret=None, langfuse_host=None, ): - if call_type not in [item.value for item in CallTypes]: - allowed_values = ", ".join([item.value for item in CallTypes]) - raise ValueError( - f"Invalid call_type {call_type}. Allowed values: {allowed_values}" - ) if messages is not None: if isinstance(messages, str): messages = [ diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index f0925110720..0e0294bbe05 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -49,6 +49,7 @@ from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter from litellm.proxy.hooks.parallel_request_limiter import ( _PROXY_MaxParallelRequestsHandler, ) +from litellm.types.utils import CallTypes if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -354,35 +355,6 @@ class ProxyLogging: print_verbose(f"final data being sent to {call_type} call: {data}") return data except Exception as e: - if "litellm_logging_obj" in data: - logging_obj: litellm.litellm_core_utils.litellm_logging.Logging = data[ - "litellm_logging_obj" - ] - - ## ASYNC FAILURE HANDLER ## - error_message = "" - if isinstance(e, HTTPException): - if isinstance(e.detail, str): - error_message = e.detail - elif isinstance(e.detail, dict): - error_message = json.dumps(e.detail) - else: - error_message = str(e) - else: - error_message = str(e) - error_raised = Exception(f"{error_message}") - await logging_obj.async_failure_handler( - exception=error_raised, - traceback_exception=traceback.format_exc(), - ) - - ## SYNC FAILURE HANDLER ## - try: - logging_obj.failure_handler( - error_raised, traceback.format_exc() - ) # DO NOT MAKE THREADED - router retry fallback relies on this! - except Exception as error_val: - pass raise e async def during_call_hook( @@ -597,12 +569,14 @@ class ProxyLogging: ) ### LOGGING ### - litellm_logging_obj: Optional[Logging] = request_data.get( - "litellm_logging_obj", None - ) - if isinstance(original_exception, HTTPException): + litellm_logging_obj: Optional[Logging] = request_data.get( + "litellm_logging_obj", None + ) if litellm_logging_obj is None: + import uuid + + request_data["litellm_call_id"] = str(uuid.uuid4()) litellm_logging_obj, data = litellm.utils.function_setup( original_function="IGNORE_THIS", rules_obj=litellm.utils.Rules(), diff --git a/litellm/tests/test_proxy_reject_logging.py b/litellm/tests/test_proxy_reject_logging.py index 7edd703815e..f176b2f8cad 100644 --- a/litellm/tests/test_proxy_reject_logging.py +++ b/litellm/tests/test_proxy_reject_logging.py @@ -23,6 +23,8 @@ import os sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path +from typing import Literal + import pytest from fastapi import Request, Response from starlette.datastructures import URL @@ -51,7 +53,20 @@ class testLogger(CustomLogger): def __init__(self): self.reaches_failure_event = False - async def async_pre_call_check(self, deployment: dict): + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: Literal[ + "completion", + "text_completion", + "embeddings", + "image_generation", + "moderation", + "audio_transcription", + ], + ): raise HTTPException( status_code=429, detail={"error": "Max parallel request limit reached"} ) @@ -92,15 +107,15 @@ router = Router( ], }, ), - # ("/v1/completions", {"model": "fake-model", "prompt": "ping"}), - # ( - # "/v1/embeddings", - # { - # "input": "The food was delicious and the waiter...", - # "model": "text-embedding-ada-002", - # "encoding_format": "float", - # }, - # ), + ("/v1/completions", {"model": "fake-model", "prompt": "ping"}), + ( + "/v1/embeddings", + { + "input": "The food was delicious and the waiter...", + "model": "text-embedding-ada-002", + "encoding_format": "float", + }, + ), ], ) @pytest.mark.asyncio diff --git a/litellm/utils.py b/litellm/utils.py index 490b809a1cf..1010beb96db 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -531,6 +531,8 @@ def function_setup( call_type == CallTypes.aspeech.value or call_type == CallTypes.speech.value ): messages = kwargs.get("input", "speech") + else: + messages = "default-message-value" stream = True if "stream" in kwargs and kwargs["stream"] == True else False logging_obj = litellm.litellm_core_utils.litellm_logging.Logging( model=model, @@ -561,10 +563,8 @@ def function_setup( ) return logging_obj, kwargs except Exception as e: - import logging - - logging.debug( - f"[Non-Blocking] {traceback.format_exc()}; args - {args}; kwargs - {kwargs}" + verbose_logger.error( + f"litellm.utils.py::function_setup() - [Non-Blocking] {traceback.format_exc()}; args - {args}; kwargs - {kwargs}" ) raise e From fd11f63f71db9e3dcb71b10e60623cd7c7f4879c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 5 Jul 2024 14:49:34 -0700 Subject: [PATCH 2/3] fix(utils.py): log failure to sync failure callbacks as well --- litellm/proxy/utils.py | 12 ++++++++++++ litellm/tests/test_proxy_reject_logging.py | 12 +++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0e0294bbe05..4665bbce7c3 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -7,6 +7,7 @@ import os import re import smtplib import subprocess +import threading import time import traceback from datetime import datetime, timedelta @@ -583,6 +584,7 @@ class ProxyLogging: start_time=datetime.now(), **request_data, ) + # log the custom exception await litellm_logging_obj.async_failure_handler( exception=original_exception, @@ -591,6 +593,16 @@ class ProxyLogging: end_time=time.time(), ) + threading.Thread( + target=litellm_logging_obj.failure_handler, + args=( + original_exception, + traceback.format_exc(), + time.time(), + time.time(), + ), + ).start() + for callback in litellm.callbacks: try: _callback: Optional[CustomLogger] = None diff --git a/litellm/tests/test_proxy_reject_logging.py b/litellm/tests/test_proxy_reject_logging.py index f176b2f8cad..b1796abee60 100644 --- a/litellm/tests/test_proxy_reject_logging.py +++ b/litellm/tests/test_proxy_reject_logging.py @@ -51,7 +51,8 @@ from litellm.router import Router class testLogger(CustomLogger): def __init__(self): - self.reaches_failure_event = False + self.reaches_sync_failure_event = False + self.reaches_async_failure_event = False async def async_pre_call_hook( self, @@ -72,10 +73,10 @@ class testLogger(CustomLogger): ) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + self.reaches_async_failure_event = True + + def log_failure_event(self, kwargs, response_obj, start_time, end_time): self.reaches_failure_event = True - return await super().async_log_failure_event( - kwargs, response_obj, start_time, end_time - ) router = Router( @@ -184,4 +185,5 @@ async def test_chat_completion_request_with_redaction(route, body): pass await asyncio.sleep(3) - assert _test_logger.reaches_failure_event is True + assert _test_logger.reaches_async_failure_event is True + assert _test_logger.reaches_sync_failure_event is True From 6090fd37fa2275e5f52f18ad071ea717baeee351 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 5 Jul 2024 19:09:37 -0700 Subject: [PATCH 3/3] fix(test_proxy_reject_logging.py): fix test --- litellm/litellm_core_utils/litellm_logging.py | 1 + litellm/proxy/utils.py | 33 ++++++++++--------- litellm/tests/test_proxy_reject_logging.py | 3 +- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c0fdd3090c2..6a8a6cfdc1b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1607,6 +1607,7 @@ class Logging: ) == False ): # custom logger class + callback.log_failure_event( start_time=start_time, end_time=end_time, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 4665bbce7c3..32b74be7c6b 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -585,23 +585,24 @@ class ProxyLogging: **request_data, ) - # log the custom exception - await litellm_logging_obj.async_failure_handler( - exception=original_exception, - traceback_exception=traceback.format_exc(), - start_time=time.time(), - end_time=time.time(), - ) + if litellm_logging_obj is not None: + # log the custom exception + await litellm_logging_obj.async_failure_handler( + exception=original_exception, + traceback_exception=traceback.format_exc(), + start_time=time.time(), + end_time=time.time(), + ) - threading.Thread( - target=litellm_logging_obj.failure_handler, - args=( - original_exception, - traceback.format_exc(), - time.time(), - time.time(), - ), - ).start() + threading.Thread( + target=litellm_logging_obj.failure_handler, + args=( + original_exception, + traceback.format_exc(), + time.time(), + time.time(), + ), + ).start() for callback in litellm.callbacks: try: diff --git a/litellm/tests/test_proxy_reject_logging.py b/litellm/tests/test_proxy_reject_logging.py index b1796abee60..865566d009b 100644 --- a/litellm/tests/test_proxy_reject_logging.py +++ b/litellm/tests/test_proxy_reject_logging.py @@ -76,7 +76,7 @@ class testLogger(CustomLogger): self.reaches_async_failure_event = True def log_failure_event(self, kwargs, response_obj, start_time, end_time): - self.reaches_failure_event = True + self.reaches_sync_failure_event = True router = Router( @@ -186,4 +186,5 @@ async def test_chat_completion_request_with_redaction(route, body): await asyncio.sleep(3) assert _test_logger.reaches_async_failure_event is True + assert _test_logger.reaches_sync_failure_event is True