From b6e0f00ed8edb7eaff1a79b2746703069c7c691a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 16:18:13 -0700 Subject: [PATCH 1/7] fix - using failure callbacks with team based logging --- litellm/proxy/proxy_config.yaml | 9 +++++++-- litellm/utils.py | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index cd625564934..5e5e9a5841c 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -9,7 +9,6 @@ model_list: model: openai/* api_key: os.environ/OPENAI_API_KEY - general_settings: store_model_in_db: true master_key: sk-1234 @@ -17,4 +16,10 @@ general_settings: litellm_settings: success_callback: ["langfuse"] - _langfuse_default_tags: ["user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] \ No newline at end of file + failure_callback: ["langfuse"] + default_team_settings: + - team_id: 7bf09cd5-217a-40d4-8634-fc31d9b88bf4 + success_callback: ["langfuse"] + failure_callback: ["langfuse"] + langfuse_public_key: "os.environ/LANGFUSE_DEVELOPMENT_PUBLIC_KEY" + langfuse_secret_key: "os.environ/LANGFUSE_DEVELOPMENT_SECRET_KEY" diff --git a/litellm/utils.py b/litellm/utils.py index 838d0fe553e..9d325b99872 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2713,6 +2713,7 @@ def function_setup( ### DYNAMIC CALLBACKS ### dynamic_success_callbacks = None dynamic_async_success_callbacks = None + dyanmic_failure_callbacks = None if kwargs.get("success_callback", None) is not None and isinstance( kwargs["success_callback"], list ): @@ -2734,6 +2735,7 @@ def function_setup( for index in reversed(removed_async_items): kwargs["success_callback"].pop(index) dynamic_success_callbacks = kwargs.pop("success_callback") + dyanmic_failure_callbacks = kwargs.pop("failure_callback") if add_breadcrumb: try: @@ -2818,7 +2820,8 @@ def function_setup( dynamic_success_callbacks=dynamic_success_callbacks, dynamic_async_success_callbacks=dynamic_async_success_callbacks, langfuse_public_key=kwargs.pop("langfuse_public_key", None), - langfuse_secret=kwargs.pop("langfuse_secret", None), + langfuse_secret=kwargs.pop("langfuse_secret", None) + or kwargs.pop("langfuse_secret_key", None), ) ## check if metadata is passed in litellm_params = {"api_base": ""} From 53f9d8280fd0eff8985b1d473602273409ee0341 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 16:37:01 -0700 Subject: [PATCH 2/7] fix - support dynamic failure callbacks --- litellm/utils.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 9d325b99872..92618e50dfa 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1079,6 +1079,7 @@ class Logging: litellm_call_id, function_id, dynamic_success_callbacks=None, + dynamic_failure_callbacks=None, dynamic_async_success_callbacks=None, langfuse_public_key=None, langfuse_secret=None, @@ -1113,7 +1114,7 @@ class Logging: self.sync_streaming_chunks = [] # for generating complete stream response self.model_call_details = {} self.dynamic_input_callbacks = [] # [TODO] callbacks set for just that call - self.dynamic_failure_callbacks = [] # [TODO] callbacks set for just that call + self.dynamic_failure_callbacks = dynamic_failure_callbacks self.dynamic_success_callbacks = ( dynamic_success_callbacks # callbacks set for just that call ) @@ -2334,11 +2335,26 @@ class Logging: start_time=start_time, end_time=end_time, ) + callbacks = [] # init this to empty incase it's not created + + if self.dynamic_failure_callbacks is not None and isinstance( + self.dynamic_failure_callbacks, list + ): + callbacks = self.dynamic_failure_callbacks + ## keep the internal functions ## + for callback in litellm.failure_callback: + if ( + isinstance(callback, CustomLogger) + and "_PROXY_" in callback.__class__.__name__ + ): + callbacks.append(callback) + else: + callbacks = litellm.failure_callback result = None # result sent to all loggers, init this to None incase it's not created self.redact_message_input_output_from_logging(result=result) - for callback in litellm.failure_callback: + for callback in callbacks: try: if callback == "lite_debugger": print_verbose("reaches lite_debugger for logging!") @@ -2713,7 +2729,7 @@ def function_setup( ### DYNAMIC CALLBACKS ### dynamic_success_callbacks = None dynamic_async_success_callbacks = None - dyanmic_failure_callbacks = None + dynamic_failure_callbacks = None if kwargs.get("success_callback", None) is not None and isinstance( kwargs["success_callback"], list ): @@ -2735,7 +2751,10 @@ def function_setup( for index in reversed(removed_async_items): kwargs["success_callback"].pop(index) dynamic_success_callbacks = kwargs.pop("success_callback") - dyanmic_failure_callbacks = kwargs.pop("failure_callback") + if kwargs.get("failure_callback", None) is not None and isinstance( + kwargs["failure_callback"], list + ): + dynamic_failure_callbacks = kwargs.pop("failure_callback") if add_breadcrumb: try: @@ -2818,6 +2837,7 @@ def function_setup( call_type=call_type, start_time=start_time, dynamic_success_callbacks=dynamic_success_callbacks, + dynamic_failure_callbacks=dynamic_failure_callbacks, dynamic_async_success_callbacks=dynamic_async_success_callbacks, langfuse_public_key=kwargs.pop("langfuse_public_key", None), langfuse_secret=kwargs.pop("langfuse_secret", None) From 92b86056cffa370571667b0faf2a1ef1a7135bec Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 16:39:49 -0700 Subject: [PATCH 3/7] fix langfuse team based logging tests --- litellm/proxy/proxy_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 5e5e9a5841c..0378efced31 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -21,5 +21,5 @@ litellm_settings: - team_id: 7bf09cd5-217a-40d4-8634-fc31d9b88bf4 success_callback: ["langfuse"] failure_callback: ["langfuse"] - langfuse_public_key: "os.environ/LANGFUSE_DEVELOPMENT_PUBLIC_KEY" - langfuse_secret_key: "os.environ/LANGFUSE_DEVELOPMENT_SECRET_KEY" + langfuse_public_key: "os.environ/LANGFUSE_DEV_PUBLIC_KEY" + langfuse_secret_key: "os.environ/LANGFUSE_DEV_SK_KEY" From ce8523808b507f314f22183b8d591ca2a7636e57 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 17:02:38 -0700 Subject: [PATCH 4/7] fix langfuse failure logging --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 92618e50dfa..de1d9ffb437 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2443,7 +2443,7 @@ class Logging: ) elif callback == "langfuse": global langFuseLogger - verbose_logger.debug("reaches langfuse for logging!") + verbose_logger.debug("reaches langfuse for logging failure") kwargs = {} for k, v in self.model_call_details.items(): if ( From 4584989a31678a2027d075b1722cdf94b973eda3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 17:33:29 -0700 Subject: [PATCH 5/7] fix - langfuse copy metadata --- litellm/integrations/langfuse.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index caf5437b242..f1640dea516 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -262,7 +262,23 @@ class LangFuseLogger: try: tags = [] - metadata = copy.deepcopy(metadata) # Avoid modifying the original metadata + try: + metadata = copy.deepcopy( + metadata + ) # Avoid modifying the original metadata + except: + new_metadata = {} + for key, value in metadata.items(): + if ( + isinstance(value, list) + or isinstance(value, dict) + or isinstance(value, str) + or isinstance(value, int) + or isinstance(value, float) + ): + new_metadata[key] = copy.deepcopy(value) + metadata = new_metadata + supports_tags = Version(langfuse.version.__version__) >= Version("2.6.3") supports_prompt = Version(langfuse.version.__version__) >= Version("2.7.3") supports_costs = Version(langfuse.version.__version__) >= Version("2.7.3") From a4695c3010f395a28e7253931cac38fc91f88057 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 17:37:32 -0700 Subject: [PATCH 6/7] test - using langfuse as a failure callback --- proxy_server_config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index 046ed7e95d5..b70d9b2022b 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -92,10 +92,12 @@ litellm_settings: default_team_settings: - team_id: team-1 success_callback: ["langfuse"] + failure_callback: ["langfuse"] langfuse_public_key: os.environ/LANGFUSE_PROJECT1_PUBLIC # Project 1 langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET # Project 1 - team_id: team-2 success_callback: ["langfuse"] + failure_callback: ["langfuse"] langfuse_public_key: os.environ/LANGFUSE_PROJECT2_PUBLIC # Project 2 langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2 From 1d25be0ca8d2a116b0985a42e50ded6dd3046af7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 17:48:44 -0700 Subject: [PATCH 7/7] fix langfuse logger re-initialized on all failure callbacks --- litellm/utils.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index de1d9ffb437..0babe2f0b10 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2452,8 +2452,16 @@ class Logging: kwargs[k] = v # this only logs streaming once, complete_streaming_response exists i.e when stream ends if langFuseLogger is None or ( - self.langfuse_public_key != langFuseLogger.public_key - and self.langfuse_secret != langFuseLogger.secret_key + ( + self.langfuse_public_key is not None + and self.langfuse_public_key + != langFuseLogger.public_key + ) + and ( + self.langfuse_public_key is not None + and self.langfuse_public_key + != langFuseLogger.public_key + ) ): langFuseLogger = LangFuseLogger( langfuse_public_key=self.langfuse_public_key,