diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index e1de57c91a0..f0e5e340041 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -1,3 +1,4 @@ +import fnmatch import hashlib import json from collections.abc import Callable, Mapping, Sequence @@ -5,7 +6,7 @@ from dataclasses import dataclass from datetime import datetime from enum import Enum from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast # noqa: TID251 # narrows untyped fallback dict shapes import litellm from litellm._logging import verbose_router_logger @@ -548,32 +549,42 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li - exact match - stripped model group match - provider-prefixed model group match + - fnmatch pattern key match (e.g. "openai/*") - generic fallback """ generic_fallback_idx: int | None = None stripped_model_fallback: list[str] | None = None + pattern_model_fallback: list[str] | None = None # mutable-ok: set in loop # rebind-ok: set in loop fallback_model_group: list[str] | None = None fallback_keys: Final = tuple(next(iter(item)) for item in fallbacks if isinstance(item, dict) and item) prefixed_model_group: Final = _provider_prefixed_model_group(model_group, fallback_keys) ## check for specific model group-specific fallbacks for idx, item in enumerate(fallbacks): if isinstance(item, dict): - fallback_key = next(iter(item)) + fallback_key = cast(str, next(iter(item))) # cast-ok: fallback keys are str if fallback_key == model_group: # check exact match fallback_model_group = item[model_group] break elif fallback_key == prefixed_model_group or _check_stripped_model_group( model_group=model_group, fallback_key=fallback_key ): # check generic fallback - stripped_model_fallback = item[fallback_key] + stripped_model_fallback = cast(list[str], item[fallback_key]) # cast-ok: chains are str lists elif fallback_key == "*": # check generic fallback generic_fallback_idx = idx + elif ( + pattern_model_fallback is None + and "*" in fallback_key + and fnmatch.fnmatchcase(model_group, fallback_key) + ): + pattern_model_fallback = cast(list[str], item[fallback_key]) # cast-ok: chains are str lists elif isinstance(item, str): fallback_model_group = [item] ## if none, check for generic fallback if fallback_model_group is None: if stripped_model_fallback is not None: fallback_model_group = stripped_model_fallback + elif pattern_model_fallback is not None: + fallback_model_group = pattern_model_fallback # rebind-ok: one branch assignment like stripped elif generic_fallback_idx is not None: fallback_model_group = fallbacks[generic_fallback_idx]["*"] diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index febb33aab7d..555e6e84eab 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -805,6 +805,40 @@ def test_get_fallback_model_group_does_not_mutate_fallbacks(): assert fallbacks == [{"gpt-3.5-turbo": ["claude-3-haiku"]}, "gpt-4o-mini"] +class TestGetFallbackModelGroupPatternKeys: + def test_provider_wildcard_key_matches_model_group(self): + fallbacks = [{"openai/*": ["claude-3-haiku"]}] + + assert get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-6-astra")[0] == [ + "claude-3-haiku" + ] + + def test_provider_wildcard_key_does_not_match_other_providers(self): + fallbacks = [{"openai/*": ["claude-3-haiku"]}] + + assert get_fallback_model_group(fallbacks=fallbacks, model_group="anthropic/claude-x")[0] is None + + def test_exact_key_beats_pattern_key(self): + fallbacks = [{"openai/*": ["generic-backup"]}, {"openai/gpt-4": ["specific-backup"]}] + + assert get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4")[0] == ["specific-backup"] + + def test_pattern_key_beats_generic_star(self): + fallbacks = [{"*": ["star-backup"]}, {"openai/*": ["openai-backup"]}] + + assert get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4")[0] == ["openai-backup"] + + def test_first_matching_pattern_key_in_list_order_wins(self): + fallbacks = [{"openai/*": ["first-backup"]}, {"*gpt*": ["second-backup"]}] + + assert get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4")[0] == ["first-backup"] + + def test_generic_star_still_applies_when_no_pattern_matches(self): + fallbacks = [{"anthropic/*": ["anthropic-backup"]}, {"*": ["star-backup"]}] + + assert get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4") == (["star-backup"], 1) + + class TestTriggerCooldownForFailedDeployment: def test_calls_set_cooldown_deployments_with_stamped_deployment_id(self): mock_router = MagicMock() diff --git a/tests/unit/test_router/test_router_fallbacks.py b/tests/unit/test_router/test_router_fallbacks.py index 88d89696297..5970355d1a8 100644 --- a/tests/unit/test_router/test_router_fallbacks.py +++ b/tests/unit/test_router/test_router_fallbacks.py @@ -427,6 +427,42 @@ def test_router_fallbacks_with_wildcard_model_name(): assert response["choices"][0]["message"]["content"] == "Hi this is claude!" +def test_router_fallbacks_with_provider_wildcard_fallback_key(): + """ + A fallback key of "openai/*" must catch a failing request routed through a + wildcard "openai/*" deployment (model_group is the requested model name, + e.g. "openai/gpt-6-astra"). + """ + router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": os.getenv("OPENAI_API_KEY"), + }, + }, + { + "model_name": "claude-3-haiku", + "litellm_params": { + "model": "claude-haiku-4-5-20251001", + "api_key": os.getenv("ANTHROPIC_API_KEY"), + "mock_response": "Hi this is claude!", + }, + }, + ], + fallbacks=[{"openai/*": ["claude-3-haiku"]}], + ) + + response = router.completion( + model="openai/gpt-6-astra", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + mock_testing_fallbacks=True, + ) + + assert response["choices"][0]["message"]["content"] == "Hi this is claude!" + + def test_get_fallback_model_group(): from litellm.router_utils.fallback_event_handlers import get_fallback_model_group diff --git a/ui/litellm-dashboard/src/app/(dashboard)/fallbacks/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/fallbacks/page.tsx new file mode 100644 index 00000000000..b93091d1986 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/fallbacks/page.tsx @@ -0,0 +1,17 @@ +"use client"; + +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks"; + +export default function FallbacksPage() { + const { accessToken, userRole, userId } = useAuthorized(); + return ( +
+
+

Fallbacks

+

Automatically retry a different model when the primary fails

+
+ +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts index f4489625dfe..0a0852bf629 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts @@ -40,6 +40,7 @@ const LEGACY_PAGE_ROUTES: ReadonlyMap = new Map( "model-insights": "model-insights", agents: "agents", "router-settings": "router-settings", + fallbacks: "fallbacks", users: "users", teams: "teams", organizations: "organizations", diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx index aca47e6b5c3..863143d3d76 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx @@ -63,8 +63,6 @@ export default function AddFallbacks({ accessToken, value = [], onChange }: AddF } }, [accessToken, isModalVisible]); - const availableModels = Array.from(new Set(modelInfo.map((option) => option.model_group))).sort(); - const handleCancel = () => { setIsModalVisible(false); // Reset to initial state @@ -125,7 +123,7 @@ export default function AddFallbacks({ accessToken, value = [], onChange }: AddF key={modalKey} groups={groups} onGroupsChange={setGroups} - availableModels={availableModels} + modelGroups={modelInfo} maxFallbacks={10} maxGroups={5} /> diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.tsx index 3893a2f47ad..efb7f3bc927 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.tsx @@ -11,6 +11,7 @@ import React, { useMemo, useState } from "react"; import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; import { toast } from "@/lib/toast"; import { AddFallbacksModal } from "./AddFallbacksModal"; +import { fallbackChainOptions, primaryModelOptions } from "./fallbackOptions"; import { FallbackGroup, FallbackGroupConfig } from "./FallbackGroupConfig"; export type FallbackEntry = { [modelName: string]: string[] }; @@ -51,10 +52,8 @@ export default function EditFallbacks({ enabled: Boolean(accessToken), }); - const availableModels = useMemo( - () => Array.from(new Set(modelGroups.map((option) => option.model_group))).sort(), - [modelGroups], - ); + const primaryOptions = useMemo(() => primaryModelOptions(modelGroups), [modelGroups]); + const chainOptions = useMemo(() => fallbackChainOptions(modelGroups), [modelGroups]); const handleSave = async () => { const primaryModel = group.primaryModel; @@ -83,7 +82,8 @@ export default function EditFallbacks({ diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx index 3897a9d9fa2..768fd0ab6a2 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx @@ -4,9 +4,11 @@ */ import { MultiSelect } from "@/components/shared/MultiSelect"; -import { SearchSelect } from "@/components/shared/SearchSelect"; +import { SearchSelect, SearchSelectOption } from "@/components/shared/SearchSelect"; +import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; import { AlertCircle, ArrowDown, X } from "lucide-react"; import React, { useId } from "react"; +import { FallbackOption } from "./fallbackOptions"; export interface FallbackGroup { id: string; @@ -17,20 +19,27 @@ export interface FallbackGroup { interface FallbackGroupConfigProps { group: FallbackGroup; onChange: (updatedGroup: FallbackGroup) => void; - availableModels: string[]; + primaryOptions: FallbackOption[]; + fallbackOptions: FallbackOption[]; maxFallbacks: number; disablePrimaryModel?: boolean; } +const withWildcardIcons = (options: FallbackOption[]): SearchSelectOption[] => + options.map((option) => + option.value.endsWith("/*") ? { ...option, icon: } : option, + ); + export function FallbackGroupConfig({ group, onChange, - availableModels, + primaryOptions, + fallbackOptions, maxFallbacks, disablePrimaryModel = false, }: FallbackGroupConfigProps) { - // Filter available options for fallbacks (exclude primary only, allow already selected to be shown for deselection) - const availableFallbackOptions = availableModels.filter((m) => m !== group.primaryModel); + // Exclude the primary only, allow already selected to be shown for deselection + const availableFallbackOptions = fallbackOptions.filter((option) => option.value !== group.primaryModel); const handlePrimaryChange = (value: string | null) => { const newFallbacks = group.fallbackModels.filter((model) => model !== value); @@ -71,7 +80,7 @@ export function FallbackGroupConfig({ ({ label: m, value: m }))} + options={withWildcardIcons(primaryOptions)} value={group.primaryModel} onValueChange={handlePrimaryChange} placeholder="Select primary model" @@ -110,7 +119,7 @@ export function FallbackGroupConfig({ {/* Add Fallback Input */}
({ label: m, value: m }))} + options={availableFallbackOptions} value={group.fallbackModels} onValueChange={handleFallbackSelect} placeholder={ diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.test.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.test.tsx index 738040149f0..49ee62c50d9 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.test.tsx @@ -6,7 +6,11 @@ import type { FallbackGroup } from "./FallbackGroupConfig"; import { toast } from "@/lib/toast"; const mockOnGroupsChange = vi.fn(); -const AVAILABLE_MODELS = ["gpt-4", "gpt-3.5-turbo", "claude-3-opus"]; +const AVAILABLE_MODEL_GROUPS = [ + { model_group: "gpt-4", providers: ["openai"] }, + { model_group: "gpt-3.5-turbo", providers: ["openai"] }, + { model_group: "claude-3-opus", providers: ["anthropic"] }, +]; describe("FallbackSelectionForm", () => { beforeEach(() => { @@ -16,14 +20,14 @@ describe("FallbackSelectionForm", () => { it("should render the component", () => { render( - , + , ); expect(screen.getByText(/no fallback groups configured/i)).toBeInTheDocument(); }); it("should show Create First Group button when no groups exist", () => { render( - , + , ); expect(screen.getByRole("button", { name: /create first group/i })).toBeInTheDocument(); }); @@ -31,7 +35,7 @@ describe("FallbackSelectionForm", () => { it("should call onGroupsChange when Create First Group is clicked", async () => { const user = userEvent.setup(); render( - , + , ); await user.click(screen.getByRole("button", { name: /create first group/i })); @@ -49,7 +53,11 @@ describe("FallbackSelectionForm", () => { it("should display tabs when groups exist", () => { const groups: FallbackGroup[] = [{ id: "1", primaryModel: null, fallbackModels: [] }]; render( - , + , ); expect(screen.getByRole("tab", { name: /group 1/i })).toBeInTheDocument(); }); @@ -57,16 +65,36 @@ describe("FallbackSelectionForm", () => { it("should display primary model as tab label when set", () => { const groups: FallbackGroup[] = [{ id: "1", primaryModel: "gpt-4", fallbackModels: [] }]; render( - , + , ); expect(screen.getByRole("tab", { name: "gpt-4" })).toBeInTheDocument(); }); + it("should display friendly label for provider wildcard primary", () => { + const groups: FallbackGroup[] = [{ id: "1", primaryModel: "openai/*", fallbackModels: [] }]; + render( + , + ); + expect(screen.getByRole("tab", { name: "All OpenAI Models" })).toBeInTheDocument(); + }); + it("should call onGroupsChange when add tab button is clicked", async () => { const user = userEvent.setup(); const groups: FallbackGroup[] = [{ id: "1", primaryModel: null, fallbackModels: [] }]; render( - , + , ); const addTabButton = screen.getByRole("button", { name: /add fallback group/i }); @@ -94,7 +122,7 @@ describe("FallbackSelectionForm", () => { , ); @@ -107,7 +135,7 @@ describe("FallbackSelectionForm", () => { , ); @@ -121,7 +149,11 @@ describe("FallbackSelectionForm", () => { { id: "2", primaryModel: "gpt-3.5-turbo", fallbackModels: [] }, ]; render( - , + , ); const removeButtons = screen.getAllByRole("button", { name: /^remove /i }); @@ -138,7 +170,11 @@ describe("FallbackSelectionForm", () => { it("should render FallbackGroupConfig for each group", () => { const groups: FallbackGroup[] = [{ id: "1", primaryModel: null, fallbackModels: [] }]; render( - , + , ); expect(screen.getByRole("combobox", { name: /primary model/i })).toHaveValue(""); expect(screen.getByText("Primary Model")).toBeInTheDocument(); @@ -147,7 +183,11 @@ describe("FallbackSelectionForm", () => { it("should display group with primary and fallback models in FallbackGroupConfig", () => { const groups: FallbackGroup[] = [{ id: "1", primaryModel: "gpt-4", fallbackModels: ["gpt-3.5-turbo"] }]; render( - , + , ); expect(screen.getByRole("tab", { name: "gpt-4" })).toBeInTheDocument(); expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); @@ -165,7 +205,7 @@ describe("FallbackSelectionForm", () => { , ); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx index aba9312cb34..17036c6c770 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx @@ -5,16 +5,18 @@ */ import { Button } from "@/components/ui/button"; +import { ModelGroup } from "@/components/llm_calls/fetch_models"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Plus, X } from "lucide-react"; -import React, { useEffect, useState } from "react"; +import React, { useEffect, useMemo, useState } from "react"; import { toast } from "@/lib/toast"; +import { fallbackChainOptions, fallbackPrimaryLabel, primaryModelOptions } from "./fallbackOptions"; import { FallbackGroup, FallbackGroupConfig } from "./FallbackGroupConfig"; interface FallbackSelectionFormProps { groups: FallbackGroup[]; onGroupsChange: (groups: FallbackGroup[]) => void; - availableModels: string[]; + modelGroups: ModelGroup[]; maxFallbacks?: number; maxGroups?: number; } @@ -22,10 +24,12 @@ interface FallbackSelectionFormProps { export function FallbackSelectionForm({ groups, onGroupsChange, - availableModels, + modelGroups, maxFallbacks = 10, maxGroups = 5, }: FallbackSelectionFormProps) { + const primaryOptions = useMemo(() => primaryModelOptions(modelGroups), [modelGroups]); + const chainOptions = useMemo(() => fallbackChainOptions(modelGroups), [modelGroups]); const [activeKey, setActiveKey] = useState(groups.length > 0 ? groups[0].id : "1"); // Reset activeKey when groups change (e.g., when modal reopens) @@ -77,7 +81,7 @@ export function FallbackSelectionForm({ }; const groupLabel = (group: FallbackGroup, index: number) => - group.primaryModel ? group.primaryModel : `Group ${index + 1}`; + group.primaryModel ? fallbackPrimaryLabel(group.primaryModel) : `Group ${index + 1}`; if (groups.length === 0) { return ( @@ -128,7 +132,8 @@ export function FallbackSelectionForm({ diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx index cdcb1eefadb..02a85e150ba 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx @@ -115,6 +115,27 @@ describe("Fallbacks", () => { ]); }); + it("should resolve a provider logo for wildcard and prefixed primary keys", async () => { + vi.mocked(networkingModule.getCallbacksCall).mockResolvedValueOnce({ + router_settings: { + fallbacks: [ + { "anthropic/*": ["gpt-4"] }, + { "openai/gpt-6-astra": ["gpt-4"] }, + { "bedrock/us.anthropic.claude-opus-4-8": ["gpt-4"] }, + ], + }, + }); + + const { container } = renderWithQueryClient(); + + await waitFor(() => { + expect(screen.getByText("All Anthropic Models")).toBeInTheDocument(); + }); + expect(screen.getByText("All Anthropic Models").parentElement).toHaveAttribute("title", "anthropic/*"); + const logos = container.querySelectorAll("img"); + expect(logos.length).toBeGreaterThanOrEqual(3); + }); + it("should render the component", async () => { renderWithQueryClient(); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx index 2ccd1a70dc5..9d71f307fd6 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx @@ -1,4 +1,5 @@ import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; +import { useQuery } from "@tanstack/react-query"; import { ArrowRight, Pencil, Play, Trash2 } from "lucide-react"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; @@ -9,6 +10,8 @@ import { ProviderLogo } from "../../../molecules/models/ProviderLogo"; import { toast } from "@/lib/toast"; import { getCallbacksCall, setCallbacksCall } from "../../../networking"; import { isProxyAdminRole } from "@/utils/roles"; +import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; +import { fallbackPrimaryLabel, resolveFallbackProvider } from "./fallbackOptions"; import AddFallbacks from "./AddFallbacks"; import EditFallbacks from "./EditFallbacks"; @@ -20,12 +23,14 @@ const modelCardClass = const iconWrapperClass = "inline-flex shrink-0 items-center justify-center px-1.5 py-1.5"; -function renderModelNameCell(modelName: string, getProviderFromModel?: (modelName: string) => string): React.ReactNode { - const provider = getProviderFromModel?.(modelName) ?? modelName; +export type ModelDisplayResolver = (modelName: string) => { provider: string; label: string; title?: string }; + +function renderModelNameCell(modelName: string, resolveModel?: ModelDisplayResolver): React.ReactNode { + const resolved = resolveModel?.(modelName) ?? { provider: modelName, label: modelName }; return ( - - - {modelName} + + + {resolved.label} ); } @@ -33,17 +38,17 @@ function renderModelNameCell(modelName: string, getProviderFromModel?: (modelNam function renderFallbacksChain( _primaryModel: string, fallbackModels: string[], - getProviderFromModel?: (modelName: string) => string, + resolveModel?: ModelDisplayResolver, ): React.ReactNode { const list = Array.isArray(fallbackModels) ? fallbackModels : []; if (list.length === 0) return null; const ChainCard = ({ modelName }: { modelName: string }) => { - const provider = getProviderFromModel?.(modelName) ?? modelName; + const resolved = resolveModel?.(modelName) ?? { provider: modelName, label: modelName }; return ( - - - {modelName} + + + {resolved.label} ); }; @@ -127,11 +132,18 @@ const Fallbacks: React.FC = ({ accessToken, userRole, userID }) const [fallbackToEdit, setFallbackToEdit] = useState(null); const { data: modelCostMapData } = useModelCostMap(); - const getProviderFromModel = (model: string): string => { - if (modelCostMapData != null && typeof modelCostMapData === "object" && model in modelCostMapData) { - return modelCostMapData[model]["litellm_provider"] ?? ""; - } - return ""; + const { data: modelGroups = [] } = useQuery({ + queryKey: ["availableModels", "fallbacks"], + queryFn: () => fetchAvailableModels(accessToken ?? ""), + enabled: Boolean(accessToken), + }); + const resolveModelDisplay = (model: string): { provider: string; label: string; title?: string } => { + const label = fallbackPrimaryLabel(model); + return { + provider: resolveFallbackProvider(model, modelCostMapData, modelGroups), + label, + ...(label !== model && { title: model }), + }; }; useEffect(() => { @@ -281,10 +293,10 @@ const Fallbacks: React.FC = ({ accessToken, userRole, userID }) Object.entries(item).map(([key, value]) => ( - {renderModelNameCell(key, getProviderFromModel)} + {renderModelNameCell(key, resolveModelDisplay)} - {renderFallbacksChain(key, Array.isArray(value) ? value : [], getProviderFromModel)} + {renderFallbacksChain(key, Array.isArray(value) ? value : [], resolveModelDisplay)} {canModify && ( diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/fallbackOptions.test.ts b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/fallbackOptions.test.ts new file mode 100644 index 00000000000..9f16691e8cd --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/fallbackOptions.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest"; +import type { ModelGroup } from "@/components/llm_calls/fetch_models"; +import { + fallbackChainOptions, + fallbackPrimaryLabel, + primaryModelOptions, + resolveFallbackProvider, +} from "./fallbackOptions"; + +const modelGroups: ModelGroup[] = [ + { model_group: "gpt-4", providers: ["openai"] }, + { model_group: "claude-3-opus", providers: ["anthropic"] }, + { model_group: "bedrock/us.anthropic.claude-opus-4-8", providers: ["bedrock"] }, +]; + +describe("primaryModelOptions", () => { + it("lists provider wildcards first with friendly labels, then model groups", () => { + const options = primaryModelOptions(modelGroups); + + expect(options.slice(0, 3)).toEqual([ + { value: "anthropic/*", label: "All Anthropic Models" }, + { value: "bedrock/*", label: "All Amazon Bedrock Models" }, + { value: "openai/*", label: "All OpenAI Models" }, + ]); + expect(options.slice(3).map((option) => option.value)).toEqual([ + "bedrock/us.anthropic.claude-opus-4-8", + "claude-3-opus", + "gpt-4", + ]); + }); + + it("derives a provider wildcard from the model group prefix when providers are absent", () => { + const options = primaryModelOptions([{ model_group: "openai/gpt-6-astra" }]); + + expect(options[0]).toEqual({ value: "openai/*", label: "All OpenAI Models" }); + expect(options).toHaveLength(2); + }); + + it("ignores prefixes that are not known provider slugs", () => { + const options = primaryModelOptions([{ model_group: "my-router/custom-tier" }]); + + expect(options).toEqual([{ value: "my-router/custom-tier", label: "my-router/custom-tier" }]); + }); + + it("folds a model group literally named openai/* into the friendly option", () => { + const options = primaryModelOptions([{ model_group: "openai/*", providers: ["openai"] }]); + + expect(options).toEqual([{ value: "openai/*", label: "All OpenAI Models" }]); + }); +}); + +describe("fallbackChainOptions", () => { + it("excludes wildcard entries, which can never be fallback targets", () => { + const options = fallbackChainOptions([...modelGroups, { model_group: "openai/*", providers: ["openai"] }]); + + expect(options.map((option) => option.value)).toEqual([ + "bedrock/us.anthropic.claude-opus-4-8", + "claude-3-opus", + "gpt-4", + ]); + }); +}); + +describe("resolveFallbackProvider", () => { + it("prefers the cost map litellm_provider", () => { + expect(resolveFallbackProvider("gpt-4", { "gpt-4": { litellm_provider: "azure" } }, modelGroups)).toBe("azure"); + }); + + it("falls back to the model group providers when the cost map misses", () => { + expect(resolveFallbackProvider("claude-3-opus", null, modelGroups)).toBe("anthropic"); + }); + + it("resolves provider wildcards and prefixed names from the prefix", () => { + expect(resolveFallbackProvider("anthropic/*", null, [])).toBe("anthropic"); + expect(resolveFallbackProvider("openai/gpt-6-astra", null, [])).toBe("openai"); + expect(resolveFallbackProvider("bedrock/us.anthropic.claude-opus-4-8", null, [])).toBe("bedrock"); + }); + + it("returns empty when nothing resolves", () => { + expect(resolveFallbackProvider("custom-group", null, [])).toBe(""); + }); +}); + +describe("fallbackPrimaryLabel", () => { + it("renders provider wildcards as All Models", () => { + expect(fallbackPrimaryLabel("openai/*")).toBe("All OpenAI Models"); + expect(fallbackPrimaryLabel("anthropic/*")).toBe("All Anthropic Models"); + }); + + it("keeps concrete names and unknown wildcards as-is", () => { + expect(fallbackPrimaryLabel("gpt-4")).toBe("gpt-4"); + expect(fallbackPrimaryLabel("custom/*")).toBe("custom/*"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/fallbackOptions.ts b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/fallbackOptions.ts new file mode 100644 index 00000000000..e28173d0f6c --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/fallbackOptions.ts @@ -0,0 +1,84 @@ +import { ModelGroup } from "@/components/llm_calls/fetch_models"; +import { getProviderLogoAndName, provider_map, resolveLitellmProviderSlug } from "@/components/provider_info_helpers"; + +export interface FallbackOption { + label: string; + value: string; +} + +const KNOWN_PROVIDER_SLUGS: ReadonlySet = new Set( + Object.values(provider_map).map((slug) => slug.toLowerCase()), +); + +const modelGroupProviderPrefix = (modelGroup: string): string | null => { + const slashIndex = modelGroup.indexOf("/"); + if (slashIndex <= 0) { + return null; + } + const prefix = modelGroup.slice(0, slashIndex).toLowerCase(); + return KNOWN_PROVIDER_SLUGS.has(prefix) ? prefix : null; +}; + +export const providerSlugsFor = (modelGroups: ModelGroup[]): string[] => { + const slugs = new Set(); + for (const modelGroup of modelGroups) { + for (const provider of modelGroup.providers ?? []) { + slugs.add(resolveLitellmProviderSlug(provider)); + } + const prefix = modelGroupProviderPrefix(modelGroup.model_group); + if (prefix) { + slugs.add(prefix); + } + } + return Array.from(slugs).sort((a, b) => a.localeCompare(b)); +}; + +export const providerWildcardOption = (slug: string): FallbackOption => { + const { displayName } = getProviderLogoAndName(slug); + return { value: `${slug}/*`, label: `All ${displayName} Models` }; +}; + +export const providerWildcardOptions = (modelGroups: ModelGroup[]): FallbackOption[] => + providerSlugsFor(modelGroups).map(providerWildcardOption); + +export const primaryModelOptions = (modelGroups: ModelGroup[]): FallbackOption[] => { + const wildcardValues = new Set(providerWildcardOptions(modelGroups).map((option) => option.value)); + const groupOptions = Array.from(new Set(modelGroups.map((modelGroup) => modelGroup.model_group))) + .filter((name) => !wildcardValues.has(name)) + .sort((a, b) => a.localeCompare(b)) + .map((name) => ({ label: name, value: name })); + return [...providerWildcardOptions(modelGroups), ...groupOptions]; +}; + +export const fallbackChainOptions = (modelGroups: ModelGroup[]): FallbackOption[] => + Array.from(new Set(modelGroups.map((modelGroup) => modelGroup.model_group))) + .filter((name) => !name.includes("*")) + .sort((a, b) => a.localeCompare(b)) + .map((name) => ({ label: name, value: name })); + +export const resolveFallbackProvider = ( + model: string, + modelCostMap: Record | null | undefined, + modelGroups: ModelGroup[], +): string => { + const fromCostMap = modelCostMap?.[model]?.litellm_provider; + if (typeof fromCostMap === "string" && fromCostMap !== "") { + return fromCostMap; + } + const provider = modelGroups.find((modelGroup) => modelGroup.model_group === model)?.providers?.[0]; + if (provider) { + return resolveLitellmProviderSlug(provider); + } + return modelGroupProviderPrefix(model) ?? ""; +}; + +export const fallbackPrimaryLabel = (model: string): string => { + if (model.endsWith("/*")) { + const slug = model.slice(0, -2); + if (KNOWN_PROVIDER_SLUGS.has(slug.toLowerCase())) { + const { displayName } = getProviderLogoAndName(slug); + return `All ${displayName} Models`; + } + } + return model; +}; diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx index 6f819607e3f..cbb16892c63 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx @@ -16,8 +16,13 @@ vi.mock("@/components/llm_calls/fetch_models", () => ({ })); vi.mock("../Settings/RouterSettings/Fallbacks/FallbackSelectionForm", () => ({ - FallbackSelectionForm: ({ availableModels }: { availableModels: string[] }) => ( -
{availableModels.join(",")}
+ FallbackSelectionForm: ({ modelGroups }: { modelGroups: { model_group: string }[] }) => ( +
+ {modelGroups + .map((modelGroup) => modelGroup.model_group) + .sort() + .join(",")} +
), })); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx index 7570806182d..f84593b0e34 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx @@ -327,8 +327,6 @@ const RouterSettingsAccordion = forwardRef option.model_group))).sort(); - // Expose getValue method via ref useImperativeHandle(ref, () => ({ getValue: () => { @@ -363,7 +361,7 @@ const RouterSettingsAccordion = forwardRef diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index 7ef951e1041..9c24e60a107 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -251,6 +251,7 @@ describe("Sidebar (leftnav)", () => { "Virtual Keys", "Playground", "Models + Endpoints", + "Fallbacks", "Agentic", "MCP Servers", "Guardrails", diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 9a6ec334b56..323bb0ad732 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -53,6 +53,7 @@ import { ScrollText, Search, Server, + Shuffle, Settings as SettingsIcon, Shield, ShieldCheck, @@ -226,6 +227,13 @@ const menuGroups: MenuGroup[] = [ icon: , roles: rolesAllowedToViewWriteScopedPages, }, + { + key: "fallbacks", + page: "fallbacks", + label: "Fallbacks", + icon: , + roles: all_admin_roles, + }, { key: "agentic", page: "agentic", diff --git a/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts b/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts index 85c7e58eaf8..fe86f92819e 100644 --- a/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts +++ b/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts @@ -666,3 +666,92 @@ describe("preferred LiteAdmin model", () => { ).toBeNull(); }); }); + +describe("router fallback operations", () => { + const routerSettings = () => ({ + routing_strategy: "least-busy", + fallbacks: [{ "gpt-4": ["claude-3-haiku"] }, { "openai/*": ["claude-3-haiku", "gpt-4"] }], + model_group_retry_policy: { "gpt-4": { AuthenticationErrorRetries: 0 } }, + }); + + beforeEach(() => { + managementFetch.mockImplementation(async (input) => { + const path = new URL(String(input)).pathname; + if (path.endsWith("/get/config/callbacks")) return json({ router_settings: routerSettings() }); + if (path.endsWith("/config/update")) return json({ status: "success" }); + throw new Error(`Unexpected request: ${path}`); + }); + }); + + const configUpdateBody = () => { + const call = managementFetch.mock.calls.find(([url]) => String(url).includes("/config/update")); + expect(call).toBeDefined(); + return JSON.parse(String(call?.[1]?.body)); + }; + + it("fallbacks_list reads router_settings.fallbacks without confirmation", async () => { + const input = options(); + const model = transport([completion([call("fallbacks_list", {})]), completion()]); + + await runLiteAdmin(input, model.client); + + expect(managementFetch).toHaveBeenCalledTimes(1); + expect(new URL(String(managementFetch.mock.calls[0][0])).pathname).toBe("/root/get/config/callbacks"); + expect(input.confirm).not.toHaveBeenCalled(); + }); + + it("fallback_set appends a new entry and saves full router_settings minus the retry policy", async () => { + const input = options(); + const model = transport([ + completion([call("fallback_set", { model: "anthropic/*", fallbacks: ["gpt-4"] })]), + completion(), + ]); + + await runLiteAdmin(input, model.client); + + expect(configUpdateBody()).toEqual({ + router_settings: { + routing_strategy: "least-busy", + fallbacks: [ + { "gpt-4": ["claude-3-haiku"] }, + { "openai/*": ["claude-3-haiku", "gpt-4"] }, + { "anthropic/*": ["gpt-4"] }, + ], + }, + }); + expect(input.confirm).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ name: "fallback_set", destructive: false }), + ); + }); + + it("fallback_set replaces the entry for an existing model in place", async () => { + const model = transport([ + completion([call("fallback_set", { model: "gpt-4", fallbacks: ["gpt-4o-mini"] })]), + completion(), + ]); + + await runLiteAdmin(options(), model.client); + + expect(configUpdateBody().router_settings.fallbacks).toEqual([ + { "gpt-4": ["gpt-4o-mini"] }, + { "openai/*": ["claude-3-haiku", "gpt-4"] }, + ]); + }); + + it("fallback_delete removes only that entry and keeps the rest of router_settings", async () => { + const input = options(); + const model = transport([completion([call("fallback_delete", { model: "openai/*" })]), completion()]); + + await runLiteAdmin(input, model.client); + + expect(configUpdateBody()).toEqual({ + router_settings: { + routing_strategy: "least-busy", + fallbacks: [{ "gpt-4": ["claude-3-haiku"] }], + }, + }); + expect(input.confirm).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ name: "fallback_delete", destructive: true }), + ); + }); +}); diff --git a/ui/litellm-dashboard/src/components/liteadmin/operations.ts b/ui/litellm-dashboard/src/components/liteadmin/operations.ts index 706b0c68be8..fbb9571c696 100644 --- a/ui/litellm-dashboard/src/components/liteadmin/operations.ts +++ b/ui/litellm-dashboard/src/components/liteadmin/operations.ts @@ -104,6 +104,22 @@ const requestLogsFields = { model: optionalText, status_filter: optional(z.enum(["success", "failure"])), }; +const fallbackTarget = text.regex(/^[^*]+$/, "Wildcard entries cannot be fallback targets."); +const fallbackSetFields = { + model: text.describe( + "The primary model group, or a provider wildcard like openai/* meaning all models from that provider.", + ), + fallbacks: z + .array(fallbackTarget) + .min(1) + .max(10) + .describe("Concrete model groups to try in order. Never a wildcard."), +}; +const fallbackDeleteFields = { model: text }; + +type RouterSettings = Record; +const fallbackEntries = (settings: RouterSettings): Record[] => + Array.isArray(settings.fallbacks) ? (settings.fallbacks as Record[]) : []; export function createLiteAdminOperations(context: OperationContext) { const active = () => { @@ -111,6 +127,14 @@ export function createLiteAdminOperations(context: OperationContext) { context.assertCurrent(); }; const auth = { accessToken: context.accessToken, signal: context.signal }; + const readRouterSettings = async (): Promise => { + const data = await apiClient.get<{ router_settings?: RouterSettings }>("/get/config/callbacks", auth); + const settings = { ...(data?.router_settings ?? {}) }; + delete settings.model_group_retry_policy; + return settings; + }; + const saveRouterSettings = (routerSettings: RouterSettings) => + apiClient.post("/config/update", { ...auth, body: { router_settings: routerSettings } }); const operation = (mode: "read" | "write" | "delete", kind: ResultKind, extraArguments?: Record) => >>( @@ -339,5 +363,39 @@ export function createLiteAdminOperations(context: OperationContext) { query: { ...a, start_date: `${a.start_date} 00:00:00`, end_date: `${a.end_date} 23:59:59` }, }), ), + operation("read", "fallback")("fallbacks_list", "List configured router fallbacks", object({}), async () => ({ + fallbacks: fallbackEntries(await readRouterSettings()), + })), + operation("write", "fallback")( + "fallback_set", + "Set the fallback chain for a model", + object(fallbackSetFields), + async (a) => { + const settings = await readRouterSettings(); + const current = fallbackEntries(settings); + const existingIndex = current.findIndex( + (entry) => entry !== null && typeof entry === "object" && a.model in entry, + ); + const updated = + existingIndex >= 0 + ? current.map((entry, index) => (index === existingIndex ? { [a.model]: a.fallbacks } : entry)) + : [...current, { [a.model]: a.fallbacks }]; + await saveRouterSettings({ ...settings, fallbacks: updated }); + return { fallbacks: updated }; + }, + ), + operation("delete", "fallback")( + "fallback_delete", + "Remove the fallback chain for a primary model", + object(fallbackDeleteFields), + async (a) => { + const settings = await readRouterSettings(); + const updated = fallbackEntries(settings) + .map((entry) => Object.fromEntries(Object.entries(entry).filter(([key]) => key !== a.model))) + .filter((entry) => Object.keys(entry).length > 0); + await saveRouterSettings({ ...settings, fallbacks: updated }); + return { fallbacks: updated }; + }, + ), ]; } diff --git a/ui/litellm-dashboard/src/components/liteadmin/toolResults.ts b/ui/litellm-dashboard/src/components/liteadmin/toolResults.ts index 355499ee0f7..90ec6fd3e0c 100644 --- a/ui/litellm-dashboard/src/components/liteadmin/toolResults.ts +++ b/ui/litellm-dashboard/src/components/liteadmin/toolResults.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -export type ResultKind = "key" | "team" | "user" | "budget" | "spend" | "log"; +export type ResultKind = "key" | "team" | "user" | "budget" | "spend" | "log" | "fallback"; const limitedList = >(schema: Schema, limit = 50) => z.preprocess((value) => (Array.isArray(value) ? value.slice(0, limit) : value), z.array(schema)); @@ -150,6 +150,7 @@ export function projectToolResult(kind: ResultKind, value: unknown, secrets: rea ]), spend: limitedList(spend), log: z.object({ ...pagination, data: limitedList(log) }), + fallback: z.object({ fallbacks: limitedList(z.record(z.string(), limitedList(text))).nullish() }), }; const parsed = schemas[kind].safeParse(value); if (!parsed.success) diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 72bed431421..eebfba9ce2c 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -9,6 +9,7 @@ export const pageDescriptions: Record = { "api-keys": "Manage virtual keys for API access and authentication", "llm-playground": "Interactive playground for testing LLM requests", models: "Configure and manage LLM models and endpoints", + fallbacks: "Configure which models to retry when the primary model fails", agents: "Create and manage AI agents", agentic: "Manage agentic resources: agents, workflow runs, and memory", workflows: "Track and inspect durable workflow run history",