From 517f5772928895ca28edf81966a9d2c162fb0919 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Apr 2024 16:07:58 -0700 Subject: [PATCH 1/2] fix - dont send alert on fail request --- litellm/model_prices_and_context_window_backup.json | 7 +++++++ litellm/proxy/utils.py | 1 + 2 files changed, 8 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 113f9413fd8..c2c172bfe8b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1535,6 +1535,13 @@ "litellm_provider": "openrouter", "mode": "chat" }, + "openrouter/meta-llama/llama-3-70b-instruct": { + "max_tokens": 8192, + "input_cost_per_token": 0.0000008, + "output_cost_per_token": 0.0000008, + "litellm_provider": "openrouter", + "mode": "chat" + }, "j2-ultra": { "max_tokens": 8192, "max_input_tokens": 8192, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 19b9a0a60f4..4fc252c3316 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -346,6 +346,7 @@ class ProxyLogging: if ( request_data is not None and request_data.get("litellm_status", "") != "success" + and request_data.get("litellm_status", "") != "fail" ): if request_data.get("deployment", None) is not None and isinstance( request_data["deployment"], dict From 8874eaa0b39fb97a3691894c2aa22cd91cd7af1f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 22 Apr 2024 16:11:04 -0700 Subject: [PATCH 2/2] fix - track litellm_status=fail --- litellm/proxy/proxy_server.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 171ab6e6479..eae9a30ed76 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3471,6 +3471,7 @@ async def completion( fastapi_response.headers["x-litellm-model-id"] = model_id return response except Exception as e: + data["litellm_status"] = "fail" # used for alerting verbose_proxy_logger.debug("EXCEPTION RAISED IN PROXY MAIN.PY") verbose_proxy_logger.debug( "\033[1;31mAn error occurred: %s\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`", @@ -3720,6 +3721,7 @@ async def chat_completion( return response except Exception as e: + data["litellm_status"] = "fail" # used for alerting traceback.print_exc() await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e @@ -3914,6 +3916,7 @@ async def embeddings( return response except Exception as e: + data["litellm_status"] = "fail" # used for alerting await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e ) @@ -4069,6 +4072,7 @@ async def image_generation( return response except Exception as e: + data["litellm_status"] = "fail" # used for alerting await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e ) @@ -4247,6 +4251,7 @@ async def audio_transcriptions( data["litellm_status"] = "success" # used for alerting return response except Exception as e: + data["litellm_status"] = "fail" # used for alerting await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e ) @@ -4408,6 +4413,7 @@ async def moderations( return response except Exception as e: + data["litellm_status"] = "fail" # used for alerting await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e )