diff --git a/litellm/router.py b/litellm/router.py index 1e9b23b2fa1..ba1645e7e5e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -7133,7 +7133,10 @@ class Router: # Check retry policy FIRST, before should_retry_this_error # This allows retry policies to override the healthy deployments check _retry_policy_applies = False - if self.retry_policy is not None or model_group_retry_policy is not None: + if ( + deployment_num_retries is None + and (self.retry_policy is not None or model_group_retry_policy is not None) + ): # get num_retries from retry policy # Use the model_group captured at the start of the function, or get it from metadata # kwargs.get("model") at this point is the deployment model, not the model_group diff --git a/litellm/router_utils/get_retry_from_policy.py b/litellm/router_utils/get_retry_from_policy.py index 7cf55e80e0c..2029b5d4d9d 100644 --- a/litellm/router_utils/get_retry_from_policy.py +++ b/litellm/router_utils/get_retry_from_policy.py @@ -8,6 +8,7 @@ from litellm.exceptions import ( AuthenticationError, BadRequestError, ContentPolicyViolationError, + InternalServerError, RateLimitError, Timeout, ) @@ -26,6 +27,7 @@ def get_num_retries_from_retry_policy( TimeoutErrorRetries: Optional[int] = None RateLimitErrorRetries: Optional[int] = None ContentPolicyViolationErrorRetries: Optional[int] = None + InternalServerErrorRetries: Optional[int] = None """ # if we can find the exception then in the retry policy -> return the number of retries @@ -50,6 +52,8 @@ def get_num_retries_from_retry_policy( return retry_policy.ContentPolicyViolationErrorRetries if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None: return retry_policy.BadRequestErrorRetries + if isinstance(exception, InternalServerError) and retry_policy.InternalServerErrorRetries is not None: + return retry_policy.InternalServerErrorRetries def reset_retry_policy() -> RetryPolicy: diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index dcd2e9edf7b..c672f3e7363 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1344,6 +1344,11 @@ def test_track_deployment_metrics(model_list): "ContentPolicyViolationError", 7, ), + ( + litellm.exceptions.InternalServerError, + "InternalServerError", + 5, + ), ], ) def test_get_num_retries_from_retry_policy( @@ -1367,6 +1372,24 @@ def test_get_num_retries_from_retry_policy( assert calc_num_retries == num_retries +def test_get_num_retries_from_retry_policy_handles_internal_server_error_directly(): + from litellm.router_utils.get_retry_from_policy import ( + get_num_retries_from_retry_policy, + ) + from litellm.types.router import RetryPolicy + + calc_num_retries = get_num_retries_from_retry_policy( + exception=litellm.InternalServerError( + message="test", + llm_provider="openai", + model="gpt-5-mini", + ), + retry_policy=RetryPolicy(InternalServerErrorRetries=5), + ) + + assert calc_num_retries == 5 + + @pytest.mark.parametrize( "exception_type, exception_name, allowed_fails", [ diff --git a/tests/test_litellm/test_router_per_deployment_num_retries.py b/tests/test_litellm/test_router_per_deployment_num_retries.py index 1bf5781c2d0..a51c308a6e1 100644 --- a/tests/test_litellm/test_router_per_deployment_num_retries.py +++ b/tests/test_litellm/test_router_per_deployment_num_retries.py @@ -3,15 +3,17 @@ Unit tests for per-deployment num_retries in litellm_params GitHub Issue: #18968 - Per-deployment max_retries/num_retries in litellm_params is not used in retry logic """ +from unittest.mock import patch + import httpx import pytest import pytest_asyncio -from unittest.mock import patch import litellm from litellm import Router -from litellm.types.router import RetryPolicy from litellm.integrations.custom_logger import CustomLogger +from litellm.router_utils.get_retry_from_policy import get_num_retries_from_retry_policy +from litellm.types.router import RetryPolicy class TestPerDeploymentNumRetries: @@ -426,6 +428,18 @@ class TestNoProviderRetryAmplification: ) assert await self._call_and_count(router) == 6 + def test_retry_policy_helper_handles_internal_server_error(self): + retry_count = get_num_retries_from_retry_policy( + exception=litellm.InternalServerError( + message="test error", + llm_provider="openai", + model="gpt-4", + ), + retry_policy=RetryPolicy(InternalServerErrorRetries=2), + ) + + assert retry_count == 2 + @pytest.mark.asyncio async def test_global_num_retries_not_amplified(self): """