diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index e1de57c91a0..f420d74f5d8 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -247,6 +247,15 @@ def _check_stripped_model_group(model_group: str, fallback_key: str) -> bool: return False +def _matches_provider_wildcard_key(model_group: str, fallback_key: str) -> bool: + provider, separator, rest = fallback_key.partition("/") + if separator != "/" or rest != "*" or not provider or "*" in provider: + return False + if model_group.startswith(f"{provider}/"): + return True + return "/" not in model_group and inferred_provider(model_group) == provider + + def _provider_prefixed_model_group(model_group: str, fallback_keys: Sequence[str]) -> str | None: if "/" in model_group or not any(key.endswith(f"/{model_group}") for key in fallback_keys): return None @@ -548,9 +557,11 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li - exact match - stripped model group match - provider-prefixed model group match + - provider wildcard match (e.g. "openai/*") - generic fallback """ generic_fallback_idx: int | None = None + provider_wildcard_fallback_idx: int | None = None stripped_model_fallback: list[str] | None = None fallback_model_group: list[str] | None = None fallback_keys: Final = tuple(next(iter(item)) for item in fallbacks if isinstance(item, dict) and item) @@ -564,20 +575,25 @@ def get_fallback_model_group(fallbacks: list[Any], model_group: str) -> tuple[li break elif fallback_key == prefixed_model_group or _check_stripped_model_group( model_group=model_group, fallback_key=fallback_key - ): # check generic fallback + ): stripped_model_fallback = item[fallback_key] + elif isinstance(fallback_key, str) and _matches_provider_wildcard_key( + model_group=model_group, fallback_key=fallback_key + ): + provider_wildcard_fallback_idx = idx elif fallback_key == "*": # check generic fallback generic_fallback_idx = idx elif isinstance(item, str): fallback_model_group = [item] - ## if none, check for generic fallback - if fallback_model_group is None: - if stripped_model_fallback is not None: - fallback_model_group = stripped_model_fallback - elif generic_fallback_idx is not None: - fallback_model_group = fallbacks[generic_fallback_idx]["*"] - - return fallback_model_group, generic_fallback_idx + if fallback_model_group is not None: + return fallback_model_group, generic_fallback_idx + if stripped_model_fallback is not None: + return stripped_model_fallback, generic_fallback_idx + if provider_wildcard_fallback_idx is not None: + return next(iter(fallbacks[provider_wildcard_fallback_idx].values())), generic_fallback_idx + if generic_fallback_idx is not None: + return fallbacks[generic_fallback_idx]["*"], generic_fallback_idx + return None, None PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file", "batch_id", "file_id", "fine_tuning_job_id") diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index febb33aab7d..9d354fcdecf 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -1463,6 +1463,80 @@ def test_get_fallback_model_group_prefixed_match_skips_prefixed_model_group(): assert fallback_model_group is None +@pytest.mark.parametrize("model_group", ["openai/gpt-4o", "gpt-4o"]) +def test_get_fallback_model_group_provider_wildcard_key_matches_every_model_of_that_provider(model_group: str): + fallbacks = [{"openai/*": ["anthropic/claude-sonnet-4-5"]}, {"*": ["gemini-2.5-flash"]}] + + fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks, model_group=model_group) + + assert fallback_model_group == ["anthropic/claude-sonnet-4-5"] + + +@pytest.mark.parametrize("model_group", ["anthropic/claude-sonnet-4-5", "claude-sonnet-4-5"]) +def test_get_fallback_model_group_provider_wildcard_key_skips_other_providers(model_group: str): + fallbacks = [{"openai/*": ["anthropic/claude-sonnet-4-5"]}, {"*": ["gemini-2.5-flash"]}] + + fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks, model_group=model_group) + + assert fallback_model_group == ["gemini-2.5-flash"] + + +def test_get_fallback_model_group_exact_key_beats_provider_wildcard_key(): + fallbacks = [{"openai/*": ["anthropic/claude-sonnet-4-5"]}, {"openai/gpt-4o": ["openai/gpt-4o-mini"]}] + + fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4o") + + assert fallback_model_group == ["openai/gpt-4o-mini"] + + +@pytest.mark.parametrize( + ("model_group", "specific_key"), + [("gpt-4o", "openai/gpt-4o"), ("openai/gpt-4o", "gpt-4o")], +) +@pytest.mark.parametrize("specific_first", [True, False]) +def test_get_fallback_model_group_per_model_key_beats_provider_wildcard_key_in_any_order( + model_group: str, specific_key: str, specific_first: bool +): + specific: Final = {specific_key: ["openai/gpt-4o-mini"]} + wildcard: Final = {"openai/*": ["anthropic/claude-sonnet-4-5"]} + fallbacks: Final = [specific, wildcard] if specific_first else [wildcard, specific] + + fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks, model_group=model_group) + + assert fallback_model_group == ["openai/gpt-4o-mini"] + + +@pytest.mark.parametrize("fallback_key", ["*" * 100 + "Z", "openai/gpt-*", "*/*", "/*"]) +def test_get_fallback_model_group_ignores_keys_that_are_not_a_single_provider_wildcard(fallback_key: str): + fallbacks: Final = [{fallback_key: ["anthropic/claude-sonnet-4-5"]}, {"*": ["gemini-2.5-flash"]}] + + fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks, model_group="openai/gpt-4o") + + assert fallback_model_group == ["gemini-2.5-flash"] + + +def test_router_falls_back_from_provider_wildcard_key(): + router = Router( + model_list=[ + { + "model_name": "openai/gpt-4o", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake", "mock_response": Exception("boom")}, + }, + { + "model_name": "anthropic/claude-sonnet-4-5", + "litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": "fake", "mock_response": "hi"}, + }, + ], + fallbacks=[{"openai/*": ["anthropic/claude-sonnet-4-5"]}], + num_retries=0, + ) + + response = router.completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "hi"}]) + + assert isinstance(response, litellm.ModelResponse) + assert response.choices[0].message.content == "hi" + + def test_get_fallback_model_group_never_resolves_a_provider_without_a_prefixed_key( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/ui/litellm-dashboard/src/app/(dashboard)/fallbacks/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/fallbacks/page.tsx new file mode 100644 index 00000000000..480f840f078 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/fallbacks/page.tsx @@ -0,0 +1,13 @@ +"use client"; + +import Fallbacks from "@/components/Settings/RouterSettings/Fallbacks/Fallbacks"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +export default function FallbacksPage() { + const { accessToken, userRole, userId } = useAuthorized(); + return ( +
+ +
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts index f4489625dfe..0a0852bf629 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts @@ -40,6 +40,7 @@ const LEGACY_PAGE_ROUTES: ReadonlyMap = new Map( "model-insights": "model-insights", agents: "agents", "router-settings": "router-settings", + fallbacks: "fallbacks", users: "users", teams: "teams", organizations: "organizations", diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx index aca47e6b5c3..d98d3133a0a 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx @@ -9,6 +9,7 @@ import { Button } from "@/components/ui/button"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { toast } from "@/lib/toast"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; +import { toModelProviders } from "./providerWildcards"; import { AddFallbacksModal } from "./AddFallbacksModal"; import { FallbackGroup } from "./FallbackGroupConfig"; import { FallbackSelectionForm } from "./FallbackSelectionForm"; @@ -126,6 +127,7 @@ export default function AddFallbacks({ accessToken, value = [], onChange }: AddF groups={groups} onGroupsChange={setGroups} availableModels={availableModels} + modelProviders={toModelProviders(modelInfo)} maxFallbacks={10} maxGroups={5} /> diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.tsx index 3893a2f47ad..25f14d37489 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/EditFallbacks.tsx @@ -9,6 +9,7 @@ import { useQuery } from "@tanstack/react-query"; import { LoaderCircle, Pencil } from "lucide-react"; import React, { useMemo, useState } from "react"; import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; +import { toModelProviders } from "./providerWildcards"; import { toast } from "@/lib/toast"; import { AddFallbacksModal } from "./AddFallbacksModal"; import { FallbackGroup, FallbackGroupConfig } from "./FallbackGroupConfig"; @@ -56,6 +57,8 @@ export default function EditFallbacks({ [modelGroups], ); + const modelProviders = useMemo(() => toModelProviders(modelGroups), [modelGroups]); + const handleSave = async () => { const primaryModel = group.primaryModel; if (!primaryModel) { @@ -84,6 +87,7 @@ export default function EditFallbacks({ group={group} onChange={setGroup} availableModels={availableModels} + modelProviders={modelProviders} maxFallbacks={maxFallbacks} disablePrimaryModel /> diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.integration.test.tsx new file mode 100644 index 00000000000..299805082ba --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.integration.test.tsx @@ -0,0 +1,59 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { useState } from "react"; +import { describe, expect, it } from "vitest"; +import { FallbackGroup, FallbackGroupConfig } from "./FallbackGroupConfig"; +import { ModelProviders } from "./providerWildcards"; + +const availableModels = ["claude-sonnet-4-5", "anthropic/claude-opus-4-1", "openai/gpt-4o", "gpt-4o-mini"]; +const modelProviders: ModelProviders = { "claude-sonnet-4-5": ["anthropic"], "gpt-4o-mini": ["openai"] }; + +function Harness({ initial, maxFallbacks = 10 }: { initial: FallbackGroup; maxFallbacks?: number }) { + const [group, setGroup] = useState(initial); + return ( + <> + + {JSON.stringify(group)} + + ); +} + +const state = (): FallbackGroup => JSON.parse(screen.getByTestId("state").textContent ?? "{}"); + +describe("FallbackGroupConfig provider wildcards", () => { + it("lets the user pick All OpenAI models as the primary, found by typing openai/*", async () => { + const user = userEvent.setup(); + render(); + + await user.type(screen.getByRole("combobox", { name: /primary model/i }), "openai/*"); + await user.click(await screen.findByRole("option", { name: /All OpenAI models/i })); + + expect(state().primaryModel).toBe("openai/*"); + }); + + it("expands All Anthropic models into every Anthropic model group in the fallback chain", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("combobox", { name: "Select fallback models to add..." })); + await user.click(await screen.findByRole("option", { name: /All Anthropic models/i })); + + expect(state().fallbackModels).toEqual(["claude-sonnet-4-5", "anthropic/claude-opus-4-1"]); + }); + + it("does not offer the primary provider wildcard as its own fallback", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("combobox", { name: "Select fallback models to add..." })); + + expect(await screen.findByRole("option", { name: /All Anthropic models/i })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: /All OpenAI models/i })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx index 3897a9d9fa2..972d443d353 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx @@ -7,6 +7,13 @@ import { MultiSelect } from "@/components/shared/MultiSelect"; import { SearchSelect } from "@/components/shared/SearchSelect"; import { AlertCircle, ArrowDown, X } from "lucide-react"; import React, { useId } from "react"; +import { + expandFallbackSelection, + fallbackModelLabel, + ModelProviders, + providerWildcardValues, + toFallbackModelOption, +} from "./providerWildcards"; export interface FallbackGroup { id: string; @@ -18,6 +25,7 @@ interface FallbackGroupConfigProps { group: FallbackGroup; onChange: (updatedGroup: FallbackGroup) => void; availableModels: string[]; + modelProviders?: ModelProviders; maxFallbacks: number; disablePrimaryModel?: boolean; } @@ -26,11 +34,17 @@ export function FallbackGroupConfig({ group, onChange, availableModels, + modelProviders = {}, maxFallbacks, disablePrimaryModel = false, }: FallbackGroupConfigProps) { - // Filter available options for fallbacks (exclude primary only, allow already selected to be shown for deselection) - const availableFallbackOptions = availableModels.filter((m) => m !== group.primaryModel); + const wildcardValues = providerWildcardValues(availableModels, modelProviders).filter( + (value) => !availableModels.includes(value), + ); + const primaryOptions = [...wildcardValues, ...availableModels].map(toFallbackModelOption); + const fallbackOptions = [...wildcardValues, ...availableModels] + .filter((m) => m !== group.primaryModel) + .map(toFallbackModelOption); const handlePrimaryChange = (value: string | null) => { const newFallbacks = group.fallbackModels.filter((model) => model !== value); @@ -42,8 +56,10 @@ export function FallbackGroupConfig({ }; const handleFallbackSelect = (values: string[]) => { - // Limit to maxFallbacks - const limitedValues = values.slice(0, maxFallbacks); + const limitedValues = expandFallbackSelection(values, availableModels, modelProviders, { + primaryModel: group.primaryModel, + maxFallbacks, + }); onChange({ ...group, @@ -71,10 +87,10 @@ export function FallbackGroupConfig({ ({ label: m, value: m }))} + options={primaryOptions} value={group.primaryModel} onValueChange={handlePrimaryChange} - placeholder="Select primary model" + placeholder="Select a model or All models" emptyText="No models found" disabled={disablePrimaryModel} className="h-12" @@ -110,7 +126,7 @@ export function FallbackGroupConfig({ {/* Add Fallback Input */}
({ label: m, value: m }))} + options={fallbackOptions} value={group.fallbackModels} onValueChange={handleFallbackSelect} placeholder={ @@ -146,7 +162,7 @@ export function FallbackGroupConfig({ {index + 1}
- {modelValue} + {fallbackModelLabel(modelValue)}
diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx index aba9312cb34..3f2895e507f 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx @@ -10,11 +10,13 @@ import { Plus, X } from "lucide-react"; import React, { useEffect, useState } from "react"; import { toast } from "@/lib/toast"; import { FallbackGroup, FallbackGroupConfig } from "./FallbackGroupConfig"; +import { ModelProviders } from "./providerWildcards"; interface FallbackSelectionFormProps { groups: FallbackGroup[]; onGroupsChange: (groups: FallbackGroup[]) => void; availableModels: string[]; + modelProviders?: ModelProviders; maxFallbacks?: number; maxGroups?: number; } @@ -23,6 +25,7 @@ export function FallbackSelectionForm({ groups, onGroupsChange, availableModels, + modelProviders, maxFallbacks = 10, maxGroups = 5, }: FallbackSelectionFormProps) { @@ -129,6 +132,7 @@ export function FallbackSelectionForm({ group={group} onChange={handleGroupUpdate} availableModels={availableModels} + modelProviders={modelProviders} maxFallbacks={maxFallbacks} /> diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx index 2ccd1a70dc5..5b2c705e816 100644 --- a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx @@ -11,6 +11,7 @@ import { getCallbacksCall, setCallbacksCall } from "../../../networking"; import { isProxyAdminRole } from "@/utils/roles"; import AddFallbacks from "./AddFallbacks"; import EditFallbacks from "./EditFallbacks"; +import { fallbackModelLabel, isProviderWildcard, wildcardProvider } from "./providerWildcards"; type FallbackEntry = { [modelName: string]: string[] }; type Fallbacks = FallbackEntry[]; @@ -20,12 +21,17 @@ const modelCardClass = const iconWrapperClass = "inline-flex shrink-0 items-center justify-center px-1.5 py-1.5"; +const providerForModel = (modelName: string, getProviderFromModel?: (modelName: string) => string): string => + isProviderWildcard(modelName) ? wildcardProvider(modelName) : getProviderFromModel?.(modelName) ?? modelName; + function renderModelNameCell(modelName: string, getProviderFromModel?: (modelName: string) => string): React.ReactNode { - const provider = getProviderFromModel?.(modelName) ?? modelName; + const provider = providerForModel(modelName, getProviderFromModel); return ( - {modelName} + + {fallbackModelLabel(modelName)} + ); } @@ -39,11 +45,13 @@ function renderFallbacksChain( if (list.length === 0) return null; const ChainCard = ({ modelName }: { modelName: string }) => { - const provider = getProviderFromModel?.(modelName) ?? modelName; + const provider = providerForModel(modelName, getProviderFromModel); return ( - {modelName} + + {fallbackModelLabel(modelName)} + ); }; diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/providerWildcards.test.ts b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/providerWildcards.test.ts new file mode 100644 index 00000000000..73a477176f7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/providerWildcards.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { + expandFallbackSelection, + fallbackModelLabel, + ModelProviders, + providerWildcardValues, + toFallbackModelOption, +} from "./providerWildcards"; + +const models = ["claude-sonnet-4-5", "anthropic/claude-opus-4-1", "openai/gpt-4o", "gpt-4o-mini"]; +const modelProviders: ModelProviders = { "claude-sonnet-4-5": ["anthropic"], "gpt-4o-mini": ["openai"] }; + +describe("providerWildcardValues", () => { + it("offers one wildcard per provider found in model metadata or a model name prefix", () => { + expect(providerWildcardValues(models, modelProviders)).toEqual(["anthropic/*", "openai/*"]); + }); +}); + +describe("fallbackModelLabel", () => { + it("names provider wildcards and the global wildcard in plain words", () => { + expect(fallbackModelLabel("openai/*")).toBe("All OpenAI models"); + expect(fallbackModelLabel("*")).toBe("All models"); + expect(fallbackModelLabel("openai/gpt-4o")).toBe("openai/gpt-4o"); + }); + + it("keeps the raw wildcard searchable as the sublabel", () => { + expect(toFallbackModelOption("openai/*")).toEqual({ + label: "All OpenAI models", + value: "openai/*", + sublabel: "openai/*", + }); + }); +}); + +describe("expandFallbackSelection", () => { + it("expands a provider wildcard into every model group of that provider", () => { + expect( + expandFallbackSelection(["anthropic/*"], models, modelProviders, { primaryModel: "openai/*", maxFallbacks: 10 }), + ).toEqual(["claude-sonnet-4-5", "anthropic/claude-opus-4-1"]); + }); + + it("drops duplicates and the primary model from the expanded chain", () => { + expect( + expandFallbackSelection(["gpt-4o-mini", "openai/*"], models, modelProviders, { + primaryModel: "openai/gpt-4o", + maxFallbacks: 10, + }), + ).toEqual(["gpt-4o-mini"]); + }); + + it("keeps a wildcard that is itself a configured model group", () => { + expect( + expandFallbackSelection(["anthropic/*"], [...models, "anthropic/*"], modelProviders, { + primaryModel: null, + maxFallbacks: 10, + }), + ).toEqual(["anthropic/*"]); + }); + + it("caps the expanded chain at the max fallback count", () => { + expect( + expandFallbackSelection(["anthropic/*"], models, modelProviders, { + primaryModel: "openai/gpt-4o", + maxFallbacks: 1, + }), + ).toEqual(["claude-sonnet-4-5"]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/providerWildcards.ts b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/providerWildcards.ts new file mode 100644 index 00000000000..e5f2a1380f6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/providerWildcards.ts @@ -0,0 +1,74 @@ +import type { ModelGroup } from "@/components/llm_calls/fetch_models"; +import { getProviderLogoAndName } from "@/components/provider_info_helpers"; + +const WILDCARD_SUFFIX = "/*"; + +export type ModelProviders = Readonly>; + +export interface FallbackModelOption { + label: string; + value: string; + sublabel?: string; +} + +export const isProviderWildcard = (value: string): boolean => + value.endsWith(WILDCARD_SUFFIX) && value.length > WILDCARD_SUFFIX.length; + +export const wildcardProvider = (value: string): string => value.slice(0, -WILDCARD_SUFFIX.length); + +export const fallbackModelLabel = (value: string): string => { + if (value === "*") { + return "All models"; + } + if (!isProviderWildcard(value)) { + return value; + } + return `All ${getProviderLogoAndName(wildcardProvider(value)).displayName} models`; +}; + +export const toModelProviders = (models: readonly ModelGroup[]): ModelProviders => + Object.fromEntries(models.map((model) => [model.model_group, model.providers ?? []])); + +const providersOf = (model: string, modelProviders: ModelProviders): readonly string[] => { + const known = modelProviders[model] ?? []; + if (known.length > 0) { + return known; + } + const [prefix, ...rest] = model.split("/"); + return rest.length > 0 && prefix ? [prefix] : []; +}; + +export const modelsForProvider = ( + provider: string, + models: readonly string[], + modelProviders: ModelProviders, +): string[] => + models.filter((model) => !isProviderWildcard(model) && providersOf(model, modelProviders).includes(provider)); + +export const providerWildcardValues = (models: readonly string[], modelProviders: ModelProviders): string[] => { + const providers = new Set( + models.filter((model) => !isProviderWildcard(model)).flatMap((model) => providersOf(model, modelProviders)), + ); + return Array.from(providers) + .sort() + .map((provider) => `${provider}${WILDCARD_SUFFIX}`); +}; + +export const toFallbackModelOption = (value: string): FallbackModelOption => + isProviderWildcard(value) ? { label: fallbackModelLabel(value), value, sublabel: value } : { label: value, value }; + +export const expandFallbackSelection = ( + values: readonly string[], + models: readonly string[], + modelProviders: ModelProviders, + { primaryModel, maxFallbacks }: { primaryModel: string | null; maxFallbacks: number }, +): string[] => { + const expanded = values.flatMap((value) => + isProviderWildcard(value) && !models.includes(value) + ? modelsForProvider(wildcardProvider(value), models, modelProviders) + : [value], + ); + return Array.from(new Set(expanded)) + .filter((model) => model !== primaryModel) + .slice(0, maxFallbacks); +}; diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index 7ef951e1041..c00c2fd27be 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -308,6 +308,10 @@ describe("Sidebar (leftnav)", () => { expect(placementsOf("router-settings")).toEqual(["SETTINGS > settings"]); }); + it("puts Fallbacks at the top level of AI GATEWAY so it is one click away", () => { + expect(placementsOf("fallbacks")).toEqual(["AI GATEWAY"]); + }); + it("has no duplicate keys among all menu items and their children", () => { // React keys must be unique across the whole nav config, otherwise the // active-item highlight and group expansion collide. diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 9a6ec334b56..33b0d33fa2f 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -54,6 +54,7 @@ import { Search, Server, Settings as SettingsIcon, + Shuffle, Shield, ShieldCheck, Tags, @@ -226,6 +227,13 @@ const menuGroups: MenuGroup[] = [ icon: , roles: rolesAllowedToViewWriteScopedPages, }, + { + key: "fallbacks", + page: "fallbacks", + label: "Fallbacks", + icon: , + roles: all_admin_roles, + }, { key: "agentic", page: "agentic", diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 72bed431421..90c8326586c 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -43,6 +43,7 @@ export const pageDescriptions: Record = { skills: "Browse and manage Claude Code skills", usage: "View legacy usage dashboard", "router-settings": "Configure routing and load balancing settings", + fallbacks: "Pick which models to try when a model fails", "logging-and-alerts": "Set up logging and alert configurations", "admin-panel": "Access admin panel and settings", };