From 2313f5f90a4f56d01406105bfe0e4adbf3d7bc8f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 6 Mar 2024 10:21:25 -0800 Subject: [PATCH 1/5] (feat) handle litellm circular ref error --- litellm/proxy/utils.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 527b66f8c0c..ddce2b221f4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -507,6 +507,18 @@ class PrismaClient: return hashed_token + def handle_circular_references(obj: Any): + try: + # Try to serialize the object using default serialization + return json.dumps(obj) + except Exception as e: + # If a TypeError is raised, return an error message JSON + error_message: dict = { + "error": "Error converting to JSON", + "error_message": str(e), + } + return json.dumps(error_message) + def jsonify_object(self, data: dict) -> dict: db_data = copy.deepcopy(data) @@ -1641,10 +1653,27 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time): if api_key is not None and isinstance(api_key, str) and api_key.startswith("sk-"): # hash the api_key api_key = hash_token(api_key) - if "headers" in metadata and "authorization" in metadata["headers"]: - metadata["headers"].pop( - "authorization" - ) # do not store the original `sk-..` api key in the db + + # clean up litellm metadata + if isinstance(metadata, dict): + clean_metadata = {} + verbose_proxy_logger.debug( + f"getting payload for SpendLogs, available keys in metadata: " + + str(list(metadata.keys())) + ) + for key in metadata: + if key in [ + "headers", + "endpoint", + "model_group", + "deployment", + "model_info", + "caching_groups", + ]: + continue + else: + clean_metadata[key] = metadata[key] + if litellm.cache is not None: cache_key = litellm.cache.get_cache_key(**kwargs) else: @@ -1668,7 +1697,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time): "team_id": kwargs.get("litellm_params", {}) .get("metadata", {}) .get("user_api_key_team_id", ""), - "metadata": metadata, + "metadata": clean_metadata, "cache_key": cache_key, "spend": kwargs.get("response_cost", 0), "total_tokens": usage.get("total_tokens", 0), From c17789f170321ab3800e4b0a036daa7a9879a140 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 6 Mar 2024 12:02:44 -0800 Subject: [PATCH 2/5] (fix) circular ref error h --- litellm/proxy/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ddce2b221f4..9a2752148ac 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1669,6 +1669,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time): "deployment", "model_info", "caching_groups", + "previous_models", ]: continue else: From ee468a4e05b710cb44fffc61ff7c75f090b4f5a5 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 6 Mar 2024 12:08:22 -0800 Subject: [PATCH 3/5] (feat) circular ref error on prisa --- litellm/proxy/utils.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9a2752148ac..1e701515e1b 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -507,24 +507,16 @@ class PrismaClient: return hashed_token - def handle_circular_references(obj: Any): - try: - # Try to serialize the object using default serialization - return json.dumps(obj) - except Exception as e: - # If a TypeError is raised, return an error message JSON - error_message: dict = { - "error": "Error converting to JSON", - "error_message": str(e), - } - return json.dumps(error_message) - def jsonify_object(self, data: dict) -> dict: db_data = copy.deepcopy(data) for k, v in db_data.items(): if isinstance(v, dict): - db_data[k] = json.dumps(v) + try: + db_data[k] = json.dumps(v) + except: + # This avoids Prisma retrying this 5 times, and making 5 clients + db_data[k] = "failed-to-serialize-json" return db_data @backoff.on_exception( From 74e50d9d351526c8b14d2d974f94804aa2103ef0 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 6 Mar 2024 12:17:59 -0800 Subject: [PATCH 4/5] (fix) high traffic langfuse logging --- litellm/integrations/langfuse.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index bb8b62b5079..8d135e8495c 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -265,8 +265,14 @@ class LangFuseLogger: cost = kwargs.get("response_cost", None) print_verbose(f"trace: {cost}") - if supports_tags: + + # Clean Metadata before logging - never log raw metadata + # the raw metadata can contain circular references which leads to infinite recursion + # we clean out all extra litellm metadata params before logging + clean_metadata = {} + if isinstance(metadata, dict): for key, value in metadata.items(): + # generate langfuse tags if key in [ "user_api_key", "user_api_key_user_id", @@ -274,6 +280,22 @@ class LangFuseLogger: "semantic-similarity", ]: tags.append(f"{key}:{value}") + + # clean litellm metadata before logging + if key in [ + "headers", + "endpoint", + "model_group", + "deployment", + "model_info", + "caching_groups", + "previous_models", + ]: + continue + else: + clean_metadata[key] = value + + if supports_tags: if "cache_hit" in kwargs: if kwargs["cache_hit"] is None: kwargs["cache_hit"] = False @@ -301,7 +323,7 @@ class LangFuseLogger: "input": input, "output": output, "usage": usage, - "metadata": metadata, + "metadata": clean_metadata, "level": level, } From c4079b2548e93b0f9d9e1b74ad22d7f1094152cb Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 6 Mar 2024 12:22:52 -0800 Subject: [PATCH 5/5] (fix) high traffic langfuse, s3 --- litellm/integrations/langfuse.py | 3 --- litellm/integrations/s3.py | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 8d135e8495c..efb5b29fe3b 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -285,9 +285,6 @@ class LangFuseLogger: if key in [ "headers", "endpoint", - "model_group", - "deployment", - "model_info", "caching_groups", "previous_models", ]: diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index e25b39777c8..dc35430bc10 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -104,6 +104,23 @@ class S3Logger: usage = response_obj["usage"] id = response_obj.get("id", str(uuid.uuid4())) + # Clean Metadata before logging - never log raw metadata + # the raw metadata can contain circular references which leads to infinite recursion + # we clean out all extra litellm metadata params before logging + clean_metadata = {} + if isinstance(metadata, dict): + for key, value in metadata.items(): + # clean litellm metadata before logging + if key in [ + "headers", + "endpoint", + "caching_groups", + "previous_models", + ]: + continue + else: + clean_metadata[key] = value + # Build the initial payload payload = { "id": id, @@ -117,7 +134,7 @@ class S3Logger: "messages": messages, "response": response_obj, "usage": usage, - "metadata": metadata, + "metadata": clean_metadata, } # Ensure everything in the payload is converted to str