scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Show global retry policy on UI (#12969)

Browse source 
* fix(router.py): return global retry policy on `get/config/callbacks`

Partial fix for https://github.com/BerriAI/litellm/issues/12855

* fix(model_dashboard.tsx): accept global retry policy

Fixes https://github.com/BerriAI/litellm/issues/12855

* fix(model_dashboard.tsx): update global retry policy, if that's what was edited
This commit is contained in:
Krish Dholakia 2025-07-24 22:37:15 -07:00 • committed by GitHub
parent 0cc320cc0a
commit dd607cd301
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 120 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
litellm/proxy/_new_secret_config.yaml
View file

@ -6,4 +6,16 @@ model_list:
litellm_settings:
callbacks: ["prometheus"]
custom_prometheus_tags: ["User-Agent: Roo-Code"]
custom_prometheus_tags: ["User-Agent: Roo-Code"]
router_settings:
routing_strategy: simple-shuffle
timeout: 300
retry_policy: {
"AuthenticationErrorRetries": 0,
"BadRequestErrorRetries": 0,
"ContentPolicyViolationErrorRetries": 0,
"InternalServerErrorRetries": 1,
"RateLimitErrorRetries": 2,
"TimeoutErrorRetries": 0
}

1
litellm/router.py
View file

@ -5988,6 +5988,7 @@ class Router:
"fallbacks",
"context_window_fallbacks",
"model_group_retry_policy",
"retry_policy",
]
for var in vars_to_include:

165
ui/litellm-dashboard/src/components/model_dashboard.tsx
View file

@ -92,6 +92,10 @@ interface RetryPolicyObject {
[key: string]: { [retryPolicyKey: string]: number } | undefined;
}
interface GlobalRetryPolicyObject {
[retryPolicyKey: string]: number;
}
interface GlobalExceptionActivityData {
sum_num_rate_limit_exceptions: number;
daily_data: { date: string; num_rate_limit_exceptions: number }[];
@ -175,6 +179,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
const [modelGroupRetryPolicy, setModelGroupRetryPolicy] =
useState<RetryPolicyObject | null>(null);
const [globalRetryPolicy, setGlobalRetryPolicy] = useState<GlobalRetryPolicyObject | null>(null);
const [defaultRetry, setDefaultRetry] = useState<number>(0);
const [globalExceptionData, setGlobalExceptionData] =
@ -445,17 +450,28 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
return;
}
console.log("new modelGroupRetryPolicy:", modelGroupRetryPolicy);
try {
const payload = {
router_settings: {
model_group_retry_policy: modelGroupRetryPolicy,
},
const payload: any = {
router_settings: {},
};
if (selectedModelGroup === "global") {
// Only update global retry policy
console.log("Saving global retry policy:", globalRetryPolicy);
if (globalRetryPolicy) {
payload.router_settings.retry_policy = globalRetryPolicy;
}
message.success("Global retry settings saved successfully");
} else {
// Only update model group retry policy
console.log("Saving model group retry policy for", selectedModelGroup, ":", modelGroupRetryPolicy);
if (modelGroupRetryPolicy) {
payload.router_settings.model_group_retry_policy = modelGroupRetryPolicy;
}
message.success(`Retry settings saved successfully for ${selectedModelGroup}`);
}
await setCallbacksCall(accessToken, payload);
message.success("Retry settings saved successfully");
} catch (error) {
console.error("Failed to save retry settings:", error);
message.error("Failed to save retry settings");
@ -627,6 +643,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
console.log("model_group_retry_policy:", model_group_retry_policy);
console.log("default_retries:", default_retries);
setModelGroupRetryPolicy(model_group_retry_policy);
setGlobalRetryPolicy(router_settings.retry_policy);
setDefaultRetry(default_retries);
} catch (error) {
console.error("There was an error fetching the model data", error);
@ -1708,50 +1725,63 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
</Grid>
</TabPanel>
<TabPanel>
<div className="flex items-center">
<Text>Filter by Public Model Name</Text>
<Select
className="mb-4 mt-2 ml-2 w-50"
defaultValue={
selectedModelGroup
? selectedModelGroup
: availableModelGroups[0]
}
value={
selectedModelGroup
? selectedModelGroup
: availableModelGroups[0]
}
onValueChange={(value) => setSelectedModelGroup(value)}
>
{availableModelGroups.map((group, idx) => (
<SelectItem
key={idx}
value={group}
onClick={() => setSelectedModelGroup(group)}
>
{group}
</SelectItem>
))}
</Select>
<div className="flex items-center gap-4 mb-6">
<div className="flex items-center">
<Text>Retry Policy Scope:</Text>
<Select
className="ml-2 w-48"
defaultValue="global"
value={selectedModelGroup === "global" ? "global" : selectedModelGroup || availableModelGroups[0]}
onValueChange={(value) => setSelectedModelGroup(value)}
>
<SelectItem value="global">Global Default</SelectItem>
{availableModelGroups.map((group, idx) => (
<SelectItem
key={idx}
value={group}
onClick={() => setSelectedModelGroup(group)}
>
{group}
</SelectItem>
))}
</Select>
</div>
</div>
<Title>Retry Policy for {selectedModelGroup}</Title>
<Text className="mb-6">
How many retries should be attempted based on the Exception
</Text>
{selectedModelGroup === "global" ? (
<>
<Title>Global Retry Policy</Title>
<Text className="mb-6">
Default retry settings applied to all model groups unless overridden
</Text>
</>
) : (
<>
<Title>Retry Policy for {selectedModelGroup}</Title>
<Text className="mb-6">
Model-specific retry settings. Falls back to global defaults if not set.
</Text>
</>
)}
{retry_policy_map && (
<table>
<tbody>
{Object.entries(retry_policy_map).map(
([exceptionType, retryPolicyKey], idx) => {
let retryCount =
modelGroupRetryPolicy?.[selectedModelGroup!]?.[
retryPolicyKey
];
if (retryCount == null) {
retryCount = defaultRetry;
let retryCount: number;
if (selectedModelGroup === "global") {
// Show global policy values
retryCount = globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry;
} else {
// Show model-group specific values with fallback to global
const modelSpecificCount = modelGroupRetryPolicy?.[selectedModelGroup!]?.[retryPolicyKey];
if (modelSpecificCount != null) {
retryCount = modelSpecificCount;
} else {
// Fall back to global policy, then default
retryCount = globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry;
}
}
return (
@ -1761,6 +1791,11 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
>
<td>
<Text>{exceptionType}</Text>
{selectedModelGroup !== "global" && (
<Text className="text-xs text-gray-500 ml-2">
(Global: {globalRetryPolicy?.[retryPolicyKey] ?? defaultRetry})
</Text>
)}
</td>
<td>
<InputNumber
@ -1769,22 +1804,34 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
min={0}
step={1}
onChange={(value) => {
setModelGroupRetryPolicy(
(prevModelGroupRetryPolicy) => {
const prevRetryPolicy =
prevModelGroupRetryPolicy?.[
selectedModelGroup!
] ?? {};
if (selectedModelGroup === "global") {
// Update global policy
setGlobalRetryPolicy((prevGlobalRetryPolicy) => {
if (value == null) return prevGlobalRetryPolicy;
return {
...(prevModelGroupRetryPolicy ??
{}),
[selectedModelGroup!]: {
...prevRetryPolicy,
[retryPolicyKey!]: value,
},
} as RetryPolicyObject;
}
);
...(prevGlobalRetryPolicy ?? {}),
[retryPolicyKey]: value,
};
});
} else {
// Update model-group specific policy
setModelGroupRetryPolicy(
(prevModelGroupRetryPolicy) => {
const prevRetryPolicy =
prevModelGroupRetryPolicy?.[
selectedModelGroup!
] ?? {};
return {
...(prevModelGroupRetryPolicy ??
{}),
[selectedModelGroup!]: {
...prevRetryPolicy,
[retryPolicyKey!]: value,
},
} as RetryPolicyObject;
}
);
}
}}
/>
</td>