From dd607cd301ea63e38a96715e10261d3b967676c4 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 24 Jul 2025 22:37:15 -0700 Subject: [PATCH] Show global retry policy on UI (#12969) * fix(router.py): return global retry policy on `get/config/callbacks` Partial fix for https://github.com/BerriAI/litellm/issues/12855 * fix(model_dashboard.tsx): accept global retry policy Fixes https://github.com/BerriAI/litellm/issues/12855 * fix(model_dashboard.tsx): update global retry policy, if that's what was edited --- litellm/proxy/_new_secret_config.yaml | 14 +- litellm/router.py | 1 + .../src/components/model_dashboard.tsx | 165 +++++++++++------- 3 files changed, 120 insertions(+), 60 deletions(-) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 108e302644f..017c1d78848 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -6,4 +6,16 @@ model_list: litellm_settings: callbacks: ["prometheus"] - custom_prometheus_tags: ["User-Agent: Roo-Code"] \ No newline at end of file + custom_prometheus_tags: ["User-Agent: Roo-Code"] + +router_settings: + routing_strategy: simple-shuffle + timeout: 300 + retry_policy: { + "AuthenticationErrorRetries": 0, + "BadRequestErrorRetries": 0, + "ContentPolicyViolationErrorRetries": 0, + "InternalServerErrorRetries": 1, + "RateLimitErrorRetries": 2, + "TimeoutErrorRetries": 0 + } \ No newline at end of file diff --git a/litellm/router.py b/litellm/router.py index acefcf93b6f..9bb8fcf6997 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5988,6 +5988,7 @@ class Router: "fallbacks", "context_window_fallbacks", "model_group_retry_policy", + "retry_policy", ] for var in vars_to_include: diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 9bdf4306cb4..3a6641c58fe 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -92,6 +92,10 @@ interface RetryPolicyObject { [key: string]: { [retryPolicyKey: string]: number } | undefined; } +interface GlobalRetryPolicyObject { + [retryPolicyKey: string]: number; +} + interface GlobalExceptionActivityData { sum_num_rate_limit_exceptions: number; daily_data: { date: string; num_rate_limit_exceptions: number }[]; @@ -175,6 +179,7 @@ const ModelDashboard: React.FC = ({ const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState(null); + const [globalRetryPolicy, setGlobalRetryPolicy] = useState(null); const [defaultRetry, setDefaultRetry] = useState(0); const [globalExceptionData, setGlobalExceptionData] = @@ -445,17 +450,28 @@ const ModelDashboard: React.FC = ({ return; } - console.log("new modelGroupRetryPolicy:", modelGroupRetryPolicy); - try { - const payload = { - router_settings: { - model_group_retry_policy: modelGroupRetryPolicy, - }, + const payload: any = { + router_settings: {}, }; + if (selectedModelGroup === "global") { + // Only update global retry policy + console.log("Saving global retry policy:", globalRetryPolicy); + if (globalRetryPolicy) { + payload.router_settings.retry_policy = globalRetryPolicy; + } + message.success("Global retry settings saved successfully"); + } else { + // Only update model group retry policy + console.log("Saving model group retry policy for", selectedModelGroup, ":", modelGroupRetryPolicy); + if (modelGroupRetryPolicy) { + payload.router_settings.model_group_retry_policy = modelGroupRetryPolicy; + } + message.success(`Retry settings saved successfully for ${selectedModelGroup}`); + } + await setCallbacksCall(accessToken, payload); - message.success("Retry settings saved successfully"); } catch (error) { console.error("Failed to save retry settings:", error); message.error("Failed to save retry settings"); @@ -627,6 +643,7 @@ const ModelDashboard: React.FC = ({ console.log("model_group_retry_policy:", model_group_retry_policy); console.log("default_retries:", default_retries); setModelGroupRetryPolicy(model_group_retry_policy); + setGlobalRetryPolicy(router_settings.retry_policy); setDefaultRetry(default_retries); } catch (error) { console.error("There was an error fetching the model data", error); @@ -1708,50 +1725,63 @@ const ModelDashboard: React.FC = ({ -
- Filter by Public Model Name - - +
+
+ Retry Policy Scope: + +
- Retry Policy for {selectedModelGroup} - - How many retries should be attempted based on the Exception - + {selectedModelGroup === "global" ? ( + <> + Global Retry Policy + + Default retry settings applied to all model groups unless overridden + + + ) : ( + <> + Retry Policy for {selectedModelGroup} + + Model-specific retry settings. Falls back to global defaults if not set. + + + )} {retry_policy_map && ( {Object.entries(retry_policy_map).map( ([exceptionType, retryPolicyKey], idx) => { - let retryCount = - modelGroupRetryPolicy?.[selectedModelGroup!]?.[ - retryPolicyKey - ]; - if (retryCount == null) { - retryCount = defaultRetry; + let retryCount: number; + + if (selectedModelGroup === "global") { + // Show global policy values + retryCount = globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry; + } else { + // Show model-group specific values with fallback to global + const modelSpecificCount = modelGroupRetryPolicy?.[selectedModelGroup!]?.[retryPolicyKey]; + if (modelSpecificCount != null) { + retryCount = modelSpecificCount; + } else { + // Fall back to global policy, then default + retryCount = globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry; + } } return ( @@ -1761,6 +1791,11 @@ const ModelDashboard: React.FC = ({ >
{exceptionType} + {selectedModelGroup !== "global" && ( + + (Global: {globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry}) + + )} = ({ min={0} step={1} onChange={(value) => { - setModelGroupRetryPolicy( - (prevModelGroupRetryPolicy) => { - const prevRetryPolicy = - prevModelGroupRetryPolicy?.[ - selectedModelGroup! - ] ?? {}; + if (selectedModelGroup === "global") { + // Update global policy + setGlobalRetryPolicy((prevGlobalRetryPolicy) => { + if (value == null) return prevGlobalRetryPolicy; return { - ...(prevModelGroupRetryPolicy ?? - {}), - [selectedModelGroup!]: { - ...prevRetryPolicy, - [retryPolicyKey!]: value, - }, - } as RetryPolicyObject; - } - ); + ...(prevGlobalRetryPolicy ?? {}), + [retryPolicyKey]: value, + }; + }); + } else { + // Update model-group specific policy + setModelGroupRetryPolicy( + (prevModelGroupRetryPolicy) => { + const prevRetryPolicy = + prevModelGroupRetryPolicy?.[ + selectedModelGroup! + ] ?? {}; + return { + ...(prevModelGroupRetryPolicy ?? + {}), + [selectedModelGroup!]: { + ...prevRetryPolicy, + [retryPolicyKey!]: value, + }, + } as RetryPolicyObject; + } + ); + } }} />