Merge pull request #25644 from michelligabriele/fix/retry-policy-internal-server-error

fix(router): honor RetryPolicy.InternalServerErrorRetries in dispatcher

docs/my-website/src/css/custom.css

@@ -899,18 +899,28 @@ video {
   font-size: 0.85rem;
 }
 
-[data-theme='dark'] .blog-wrapper article header h1,
-[data-theme='dark'] .blog-wrapper article .markdown h2,
-[data-theme='dark'] .blog-wrapper article .markdown h3 {
+[data-theme='dark'].blog-wrapper article header h1,
+[data-theme='dark'].blog-wrapper article .markdown h2,
+[data-theme='dark'].blog-wrapper article .markdown h3 {
   color: #f9fafb;
 }
 
-[data-theme='dark'] .blog-wrapper article .markdown {
+[data-theme='dark'].blog-wrapper article .markdown {
   color: #d1d5db;
 }
 
-[data-theme='dark'] .blog-wrapper article .markdown code {
+[data-theme='dark'].blog-wrapper article .markdown code {
   background: #1f2937;
   border-color: #374151;
   color: #f9fafb;
 }
+
+[data-theme='dark'].blog-wrapper article .markdown pre {
+  background: #161b22 !important;
+  border-color: #30363d !important;
+  box-shadow: none;
+}
+
+[data-theme='dark'].blog-wrapper article .markdown a {
+  color: #38bdf8;
+}

docs/my-website/src/theme/BlogListPage/styles.module.css

@@ -252,3 +252,32 @@
 [data-theme='dark'] .hiringBtn:hover {
   background: #fff;
 }
+
+[data-theme='dark'] .marqueeItem {
+  color: #9ca3af;
+}
+
+[data-theme='dark'] .marqueeSep {
+  color: #374151;
+}
+
+[data-theme='dark'] .marqueeLabel {
+  color: #6b7280;
+}
+
+[data-theme='dark'] .pageLink {
+  color: #d1d5db;
+}
+
+[data-theme='dark'] .pageLink:hover {
+  color: #38bdf8;
+}
+
+[data-theme='dark'] .meta {
+  color: #9ca3af;
+}
+
+[data-theme='dark'] .metaDash,
+[data-theme='dark'] .authorSep {
+  color: #4b5563;
+}

litellm/router_utils/get_retry_from_policy.py

@@ -28,7 +28,13 @@ def get_num_retries_from_retry_policy(
     TimeoutErrorRetries: Optional[int] = None
     RateLimitErrorRetries: Optional[int] = None
     ContentPolicyViolationErrorRetries: Optional[int] = None
+    InternalServerErrorRetries: Optional[int] = None
     """
+    # Lazy import: `InternalServerError` is defined late in `litellm/exceptions.py`,
+    # after a cyclic import re-enters this module during exceptions.py evaluation.
+    # A module-level import would be flagged by CodeQL as potentially undefined.
+    from litellm.exceptions import InternalServerError
+
     # if we can find the exception then in the retry policy -> return the number of retries
 
     if (
@@ -55,6 +61,11 @@ def get_num_retries_from_retry_policy(
         and retry_policy.RateLimitErrorRetries is not None
     ):
         return retry_policy.RateLimitErrorRetries
+    if (
+        isinstance(exception, InternalServerError)
+        and retry_policy.InternalServerErrorRetries is not None
+    ):
+        return retry_policy.InternalServerErrorRetries
     if (
         isinstance(exception, ContentPolicyViolationError)
         and retry_policy.ContentPolicyViolationErrorRetries is not None

tests/litellm/router_utils/test_get_retry_from_policy.py

@@ -0,0 +1,39 @@
+"""Unit tests for litellm.router_utils.get_retry_from_policy."""
+
+import litellm
+from litellm.router_utils.get_retry_from_policy import (
+    get_num_retries_from_retry_policy,
+)
+from litellm.types.router import RetryPolicy
+
+
+def test_internal_server_error_retries_is_honored():
+    """Regression: `InternalServerErrorRetries` must be returned for
+    `InternalServerError` exceptions. Previously the dispatcher had no
+    branch for this field and silently returned `None`, causing the
+    caller to fall back to `num_retries`."""
+    retry_policy = RetryPolicy(InternalServerErrorRetries=5)
+    exc = litellm.exceptions.InternalServerError(
+        message="test", llm_provider="openai", model="gpt-3.5-turbo"
+    )
+
+    num_retries = get_num_retries_from_retry_policy(
+        exception=exc, retry_policy=retry_policy
+    )
+
+    assert num_retries == 5
+
+
+def test_internal_server_error_retries_unset_returns_none():
+    """When the field is not set, the dispatcher should return `None`
+    so the caller falls back to `num_retries`."""
+    retry_policy = RetryPolicy()
+    exc = litellm.exceptions.InternalServerError(
+        message="test", llm_provider="openai", model="gpt-3.5-turbo"
+    )
+
+    num_retries = get_num_retries_from_retry_policy(
+        exception=exc, retry_policy=retry_policy
+    )
+
+    assert num_retries is None

tests/router_unit_tests/test_router_helper_utils.py

@@ -957,6 +957,7 @@ def test_track_deployment_metrics(model_list):
             "ContentPolicyViolationError",
             7,
         ),
+        (litellm.exceptions.InternalServerError, "InternalServerError", 5),
     ],
 )
 def test_get_num_retries_from_retry_policy(