From 8d0cd8a37f33bb0e8422e24b71071c95e527b079 Mon Sep 17 00:00:00 2001 From: 72004 Date: Fri, 21 Aug 2026 17:21:13 +0500 Subject: [PATCH 1/4] fix(router): handle InternalServerError in get_num_retries_from_retry_policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RetryPolicy.InternalServerErrorRetries was accepted in config and validated by pydantic but never read at runtime — the resolver had branches for five error types and omitted InternalServerError. Operators who configured per-error 5xx retries silently fell through to the global num_retries default. Added the missing isinstance branch and a regression test. Fixes #37805 --- litellm/router_utils/get_retry_from_policy.py | 4 ++++ tests/router_unit_tests/test_router_helper_utils.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/litellm/router_utils/get_retry_from_policy.py b/litellm/router_utils/get_retry_from_policy.py index 7cf55e80e0c..2029b5d4d9d 100644 --- a/litellm/router_utils/get_retry_from_policy.py +++ b/litellm/router_utils/get_retry_from_policy.py @@ -8,6 +8,7 @@ from litellm.exceptions import ( AuthenticationError, BadRequestError, ContentPolicyViolationError, + InternalServerError, RateLimitError, Timeout, ) @@ -26,6 +27,7 @@ def get_num_retries_from_retry_policy( TimeoutErrorRetries: Optional[int] = None RateLimitErrorRetries: Optional[int] = None ContentPolicyViolationErrorRetries: Optional[int] = None + InternalServerErrorRetries: Optional[int] = None """ # if we can find the exception then in the retry policy -> return the number of retries @@ -50,6 +52,8 @@ def get_num_retries_from_retry_policy( return retry_policy.ContentPolicyViolationErrorRetries if isinstance(exception, BadRequestError) and retry_policy.BadRequestErrorRetries is not None: return retry_policy.BadRequestErrorRetries + if isinstance(exception, InternalServerError) and retry_policy.InternalServerErrorRetries is not None: + return retry_policy.InternalServerErrorRetries def reset_retry_policy() -> RetryPolicy: diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index f81578dbd99..62ccfa44f66 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1354,6 +1354,11 @@ def test_track_deployment_metrics(model_list): "ContentPolicyViolationError", 7, ), + ( + litellm.exceptions.InternalServerError, + "InternalServerError", + 5, + ), ], ) def test_get_num_retries_from_retry_policy( From a682ac09aee2f913cebf3a564361a540501b08c2 Mon Sep 17 00:00:00 2001 From: 72004 Date: Fri, 21 Aug 2026 18:46:35 +0500 Subject: [PATCH 2/4] fix(router): preserve deployment retry budget with policy --- litellm/router.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index e9eeab53934..f6b982f3f62 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6703,7 +6703,10 @@ class Router: # Check retry policy FIRST, before should_retry_this_error # This allows retry policies to override the healthy deployments check _retry_policy_applies = False - if self.retry_policy is not None or model_group_retry_policy is not None: + if ( + deployment_num_retries is None + and (self.retry_policy is not None or model_group_retry_policy is not None) + ): # get num_retries from retry policy # Use the model_group captured at the start of the function, or get it from metadata # kwargs.get("model") at this point is the deployment model, not the model_group From f2d40563d452109d39861bf7179e9ebf23b1b6ee Mon Sep 17 00:00:00 2001 From: 72004 Date: Fri, 21 Aug 2026 19:25:22 +0500 Subject: [PATCH 3/4] test(router): cover internal server retry policy --- .../test_router_per_deployment_num_retries.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/test_router_per_deployment_num_retries.py b/tests/test_litellm/test_router_per_deployment_num_retries.py index 1bf5781c2d0..a51c308a6e1 100644 --- a/tests/test_litellm/test_router_per_deployment_num_retries.py +++ b/tests/test_litellm/test_router_per_deployment_num_retries.py @@ -3,15 +3,17 @@ Unit tests for per-deployment num_retries in litellm_params GitHub Issue: #18968 - Per-deployment max_retries/num_retries in litellm_params is not used in retry logic """ +from unittest.mock import patch + import httpx import pytest import pytest_asyncio -from unittest.mock import patch import litellm from litellm import Router -from litellm.types.router import RetryPolicy from litellm.integrations.custom_logger import CustomLogger +from litellm.router_utils.get_retry_from_policy import get_num_retries_from_retry_policy +from litellm.types.router import RetryPolicy class TestPerDeploymentNumRetries: @@ -426,6 +428,18 @@ class TestNoProviderRetryAmplification: ) assert await self._call_and_count(router) == 6 + def test_retry_policy_helper_handles_internal_server_error(self): + retry_count = get_num_retries_from_retry_policy( + exception=litellm.InternalServerError( + message="test error", + llm_provider="openai", + model="gpt-4", + ), + retry_policy=RetryPolicy(InternalServerErrorRetries=2), + ) + + assert retry_count == 2 + @pytest.mark.asyncio async def test_global_num_retries_not_amplified(self): """ From 3d8adc9925dc2127b69da1c280d479e617df2eeb Mon Sep 17 00:00:00 2001 From: 72004 Date: Fri, 21 Aug 2026 19:36:20 +0500 Subject: [PATCH 4/4] test(router): cover internal server retry helper directly --- .../test_router_helper_utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index 62ccfa44f66..108c4578027 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1382,6 +1382,24 @@ def test_get_num_retries_from_retry_policy( assert calc_num_retries == num_retries +def test_get_num_retries_from_retry_policy_handles_internal_server_error_directly(): + from litellm.router_utils.get_retry_from_policy import ( + get_num_retries_from_retry_policy, + ) + from litellm.types.router import RetryPolicy + + calc_num_retries = get_num_retries_from_retry_policy( + exception=litellm.InternalServerError( + message="test", + llm_provider="openai", + model="gpt-5-mini", + ), + retry_policy=RetryPolicy(InternalServerErrorRetries=5), + ) + + assert calc_num_retries == 5 + + @pytest.mark.parametrize( "exception_type, exception_name, allowed_fails", [