From 0f03e53348b541b9ee5e15c168f60003fd8e83e1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 4 May 2024 18:50:38 -0700 Subject: [PATCH 1/5] feat return num retries in exceptions --- litellm/utils.py | 124 +++++++++++++++++++++++------------------------ 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index ae697fe18c9..f5c0fd44787 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7886,21 +7886,51 @@ def exception_type( exception_type = type(original_exception).__name__ else: exception_type = "" - _api_base = "" - try: - _api_base = litellm.get_api_base( - model=model, optional_params=extra_kwargs - ) - except: - _api_base = "" - error_str += f" \n model: {model} \n api_base: {_api_base} \n" - error_str += str(completion_kwargs) + ################################################################################ + # Common Extra information needed for all providers + # We pass num retries, api_base, vertex_deployment etc to the exception here + ################################################################################ + + _api_base = litellm.get_api_base(model=model, optional_params=extra_kwargs) + messages = litellm.get_first_chars_messages(kwargs=completion_kwargs) + _previous_requests = extra_kwargs.get("previous_models", None) + _vertex_project = extra_kwargs.get("vertex_project") + _vertex_location = extra_kwargs.get("vertex_location") + _metadata = extra_kwargs.get("metadata", {}) or {} + _model_group = _metadata.get("model_group") + _deployment = _metadata.get("deployment") + num_retries = 0 + if _previous_requests and isinstance(_previous_requests, list): + num_retries = len(_previous_requests) + extra_information = f"\nModel: {model}" + if _api_base: + extra_information += f"\nAPI Base: {_api_base}" + if messages and len(messages) > 0: + extra_information += f"\nMessages: {messages}" + extra_information += f"\nNum Retries: {num_retries}" + + if _model_group is not None: + extra_information += f"\nmodel_group: {_model_group}\n" + if _deployment is not None: + extra_information += f"\ndeployment: {_deployment}\n" + if _vertex_project is not None: + extra_information += f"\nvertex_project: {_vertex_project}\n" + if _vertex_location is not None: + extra_information += f"\nvertex_location: {_vertex_location}\n" + + ################################################################################ + # End of Common Extra information Needed for all providers + ################################################################################ + + ################################################################################ + #################### Start of Provider Exception mapping #################### + ################################################################################ if "Request Timeout Error" in error_str or "Request timed out" in error_str: exception_mapping_worked = True raise Timeout( - message=f"APITimeoutError - Request timed out. \n model: {model} \n api_base: {_api_base} \n error_str: {error_str}", + message=f"APITimeoutError - Request timed out. {extra_information} \n error_str: {error_str}", model=model, llm_provider=custom_llm_provider, ) @@ -7935,7 +7965,7 @@ def exception_type( ): exception_mapping_worked = True raise ContextWindowExceededError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", llm_provider=custom_llm_provider, model=model, response=original_exception.response, @@ -7946,7 +7976,7 @@ def exception_type( ): exception_mapping_worked = True raise NotFoundError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", llm_provider=custom_llm_provider, model=model, response=original_exception.response, @@ -7957,7 +7987,7 @@ def exception_type( ): exception_mapping_worked = True raise ContentPolicyViolationError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", llm_provider=custom_llm_provider, model=model, response=original_exception.response, @@ -7968,7 +7998,7 @@ def exception_type( ): exception_mapping_worked = True raise BadRequestError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", llm_provider=custom_llm_provider, model=model, response=original_exception.response, @@ -7979,7 +8009,7 @@ def exception_type( ): exception_mapping_worked = True raise AuthenticationError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", llm_provider=custom_llm_provider, model=model, response=original_exception.response, @@ -7991,7 +8021,7 @@ def exception_type( ) raise APIError( status_code=500, - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", llm_provider=custom_llm_provider, model=model, request=_request, @@ -8001,7 +8031,7 @@ def exception_type( if original_exception.status_code == 401: exception_mapping_worked = True raise AuthenticationError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", llm_provider=custom_llm_provider, model=model, response=original_exception.response, @@ -8009,7 +8039,7 @@ def exception_type( elif original_exception.status_code == 404: exception_mapping_worked = True raise NotFoundError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", model=model, llm_provider=custom_llm_provider, response=original_exception.response, @@ -8017,14 +8047,14 @@ def exception_type( elif original_exception.status_code == 408: exception_mapping_worked = True raise Timeout( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", model=model, llm_provider=custom_llm_provider, ) elif original_exception.status_code == 422: exception_mapping_worked = True raise BadRequestError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", model=model, llm_provider=custom_llm_provider, response=original_exception.response, @@ -8032,7 +8062,7 @@ def exception_type( elif original_exception.status_code == 429: exception_mapping_worked = True raise RateLimitError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", model=model, llm_provider=custom_llm_provider, response=original_exception.response, @@ -8040,7 +8070,7 @@ def exception_type( elif original_exception.status_code == 503: exception_mapping_worked = True raise ServiceUnavailableError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", model=model, llm_provider=custom_llm_provider, response=original_exception.response, @@ -8048,7 +8078,7 @@ def exception_type( elif original_exception.status_code == 504: # gateway timeout error exception_mapping_worked = True raise Timeout( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", model=model, llm_provider=custom_llm_provider, ) @@ -8056,7 +8086,7 @@ def exception_type( exception_mapping_worked = True raise APIError( status_code=original_exception.status_code, - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", llm_provider=custom_llm_provider, model=model, request=original_exception.request, @@ -8064,7 +8094,7 @@ def exception_type( else: # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors raise APIConnectionError( - message=f"{exception_provider} - {message}", + message=f"{exception_provider} - {message} {extra_information}", llm_provider=custom_llm_provider, model=model, request=httpx.Request( @@ -8371,33 +8401,13 @@ def exception_type( response=original_exception.response, ) elif custom_llm_provider == "vertex_ai": - if completion_kwargs is not None: - # add model, deployment and model_group to the exception message - _model = completion_kwargs.get("model") - error_str += f"\nmodel: {_model}\n" - if extra_kwargs is not None: - _vertex_project = extra_kwargs.get("vertex_project") - _vertex_location = extra_kwargs.get("vertex_location") - _metadata = extra_kwargs.get("metadata", {}) or {} - _model_group = _metadata.get("model_group") - _deployment = _metadata.get("deployment") - - if _model_group is not None: - error_str += f"model_group: {_model_group}\n" - if _deployment is not None: - error_str += f"deployment: {_deployment}\n" - if _vertex_project is not None: - error_str += f"vertex_project: {_vertex_project}\n" - if _vertex_location is not None: - error_str += f"vertex_location: {_vertex_location}\n" - if ( "Vertex AI API has not been used in project" in error_str or "Unable to find your project" in error_str ): exception_mapping_worked = True raise BadRequestError( - message=f"VertexAIException - {error_str}", + message=f"VertexAIException - {error_str} {extra_information}", model=model, llm_provider="vertex_ai", response=original_exception.response, @@ -8408,7 +8418,7 @@ def exception_type( ): exception_mapping_worked = True raise APIError( - message=f"VertexAIException - {error_str}", + message=f"VertexAIException - {error_str} {extra_information}", status_code=500, model=model, llm_provider="vertex_ai", @@ -8417,7 +8427,7 @@ def exception_type( elif "403" in error_str: exception_mapping_worked = True raise BadRequestError( - message=f"VertexAIException - {error_str}", + message=f"VertexAIException - {error_str} {extra_information}", model=model, llm_provider="vertex_ai", response=original_exception.response, @@ -8425,7 +8435,7 @@ def exception_type( elif "The response was blocked." in error_str: exception_mapping_worked = True raise UnprocessableEntityError( - message=f"VertexAIException - {error_str}", + message=f"VertexAIException - {error_str} {extra_information}", model=model, llm_provider="vertex_ai", response=httpx.Response( @@ -8444,7 +8454,7 @@ def exception_type( ): exception_mapping_worked = True raise RateLimitError( - message=f"VertexAIException - {error_str}", + message=f"VertexAIException - {error_str} {extra_information}", model=model, llm_provider="vertex_ai", response=httpx.Response( @@ -8459,7 +8469,7 @@ def exception_type( if original_exception.status_code == 400: exception_mapping_worked = True raise BadRequestError( - message=f"VertexAIException - {error_str}", + message=f"VertexAIException - {error_str} {extra_information}", model=model, llm_provider="vertex_ai", response=original_exception.response, @@ -8467,7 +8477,7 @@ def exception_type( if original_exception.status_code == 500: exception_mapping_worked = True raise APIError( - message=f"VertexAIException - {error_str}", + message=f"VertexAIException - {error_str} {extra_information}", status_code=500, model=model, llm_provider="vertex_ai", @@ -9061,16 +9071,6 @@ def exception_type( request=original_exception.request, ) elif custom_llm_provider == "azure": - _api_base = litellm.get_api_base( - model=model, optional_params=extra_kwargs - ) - messages = litellm.get_first_chars_messages(kwargs=completion_kwargs) - extra_information = f"\nModel: {model}" - if _api_base: - extra_information += f"\nAPI Base: {_api_base}" - if messages and len(messages) > 0: - extra_information += f"\nMessages: {messages}" - if "Internal server error" in error_str: exception_mapping_worked = True raise APIError( From 5be8c95c6e715ba3ff9dd88ccb43ad8895f39db6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 4 May 2024 19:07:28 -0700 Subject: [PATCH 2/5] fix don't return num retries in utils.py --- litellm/utils.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index f5c0fd44787..0565362e54f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7894,21 +7894,16 @@ def exception_type( _api_base = litellm.get_api_base(model=model, optional_params=extra_kwargs) messages = litellm.get_first_chars_messages(kwargs=completion_kwargs) - _previous_requests = extra_kwargs.get("previous_models", None) _vertex_project = extra_kwargs.get("vertex_project") _vertex_location = extra_kwargs.get("vertex_location") _metadata = extra_kwargs.get("metadata", {}) or {} _model_group = _metadata.get("model_group") _deployment = _metadata.get("deployment") - num_retries = 0 - if _previous_requests and isinstance(_previous_requests, list): - num_retries = len(_previous_requests) extra_information = f"\nModel: {model}" if _api_base: extra_information += f"\nAPI Base: {_api_base}" if messages and len(messages) > 0: extra_information += f"\nMessages: {messages}" - extra_information += f"\nNum Retries: {num_retries}" if _model_group is not None: extra_information += f"\nmodel_group: {_model_group}\n" From 6b59aeb603672b27f1c7170f201b884dc8b212f7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 4 May 2024 19:09:34 -0700 Subject: [PATCH 3/5] fix return num retries in exceptions --- litellm/router.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 3b1c1d1022a..40671fdb9dd 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1544,6 +1544,10 @@ class Router: num_retries=num_retries, ) await asyncio.sleep(_timeout) + try: + original_exception.message += f"\nNumber Retries = {current_attempt}" + except: + pass raise original_exception def function_with_fallbacks(self, *args, **kwargs): @@ -1703,7 +1707,7 @@ class Router: response = original_function(*args, **kwargs) return response - except Exception as e: + except Exception as most_recent_exception: ## LOGGING kwargs = self.log_retry(kwargs=kwargs, e=e) remaining_retries = num_retries - current_attempt @@ -1713,7 +1717,7 @@ class Router: num_retries=num_retries, ) time.sleep(_timeout) - raise original_exception + raise most_recent_exception ### HELPER FUNCTIONS @@ -1848,7 +1852,7 @@ class Router: exception_status = 500 _should_retry = litellm._should_retry(status_code=exception_status) - if updated_fails > self.allowed_fails or _should_retry == False: + if updated_fails > self.allowed_fails: # get the current cooldown list for that minute cooldown_key = f"{current_minute}:cooldown_models" # group cooldown models by minute to reduce number of redis calls cached_value = self.cache.get_cache(key=cooldown_key) From dfc22194b2ab27c3d2d572ef312e374c0629c73b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 4 May 2024 19:11:57 -0700 Subject: [PATCH 4/5] fix - undo local dev changes --- litellm/router.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 40671fdb9dd..c4e407a2502 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1707,7 +1707,7 @@ class Router: response = original_function(*args, **kwargs) return response - except Exception as most_recent_exception: + except Exception as e: ## LOGGING kwargs = self.log_retry(kwargs=kwargs, e=e) remaining_retries = num_retries - current_attempt @@ -1717,7 +1717,7 @@ class Router: num_retries=num_retries, ) time.sleep(_timeout) - raise most_recent_exception + raise original_exception ### HELPER FUNCTIONS @@ -1852,7 +1852,7 @@ class Router: exception_status = 500 _should_retry = litellm._should_retry(status_code=exception_status) - if updated_fails > self.allowed_fails: + if updated_fails > self.allowed_fails or _should_retry == False: # get the current cooldown list for that minute cooldown_key = f"{current_minute}:cooldown_models" # group cooldown models by minute to reduce number of redis calls cached_value = self.cache.get_cache(key=cooldown_key) From 95864a1d37d492f32cfbeb01e9574477afde9cee Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 4 May 2024 20:24:15 -0700 Subject: [PATCH 5/5] fix router debug logs --- litellm/tests/test_router_debug_logs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_router_debug_logs.py b/litellm/tests/test_router_debug_logs.py index 9c0a76fc7ac..d9c8d4e69b0 100644 --- a/litellm/tests/test_router_debug_logs.py +++ b/litellm/tests/test_router_debug_logs.py @@ -82,7 +82,7 @@ def test_async_fallbacks(caplog): # Define the expected log messages # - error request, falling back notice, success notice expected_logs = [ - "litellm.acompletion(model=gpt-3.5-turbo)\x1b[31m Exception OpenAIException - Error code: 401 - {'error': {'message': 'Incorrect API key provided: bad-key. You can find your API key at https://platform.openai.com/account/api-keys.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_api_key'}}\x1b[0m", + "litellm.acompletion(model=gpt-3.5-turbo)\x1b[31m Exception OpenAIException - Error code: 401 - {'error': {'message': 'Incorrect API key provided: bad-key. You can find your API key at https://platform.openai.com/account/api-keys.', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_api_key'}} \nModel: gpt-3.5-turbo\nAPI Base: https://api.openai.com\nMessages: [{'content': 'Hello, how are you?', 'role': 'user'}]\nmodel_group: gpt-3.5-turbo\n\ndeployment: gpt-3.5-turbo\n\x1b[0m", "litellm.acompletion(model=None)\x1b[31m Exception No deployments available for selected model, passed model=gpt-3.5-turbo\x1b[0m", "Falling back to model_group = azure/gpt-3.5-turbo", "litellm.acompletion(model=azure/chatgpt-v-2)\x1b[32m 200 OK\x1b[0m",