diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index 8069e7a0504..8c6dae07ddc 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -210,6 +210,25 @@ describe("AddAutoRouterTab", () => { expect(toast.success).toHaveBeenCalledWith(`Configured with ${openAiPreset.label}`); }); + it("mixes available models from the preferred tier catalog when no complete template fits", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["gpt-5.6-luna", "claude-sonnet-5", "gpt-5.6-sol"].map((model_group) => ({ + model_group, + mode: "chat", + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([]); + renderWithProviders(); + + const button = screen.getByTestId("configure-automatically-button"); + await waitFor(() => expect(button).toBeEnabled()); + await userEvent.click(button); + + expect( + screen.getByText(/Simple: gpt-5.6-luna.*Medium: claude-sonnet-5.*Complex: gpt-5.6-sol.*Reasoning: gpt-5.6-sol/), + ).toBeInTheDocument(); + }); + // Nothing is filled in, so there is nothing to submit. The button reports that itself instead of // accepting a click and answering with a toast. it("offers no submit at all until every tier has a model", async () => { diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index d671403717b..8510e1ad8c2 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -65,7 +65,7 @@ import { } from "@/lib/autorouter_presets"; import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; -import { buildAutomaticRouterConfig } from "./auto_setup"; +import { buildAutomaticRouterConfig, buildPreferredTierModels } from "./auto_setup"; interface AddAutoRouterTabProps { handleOk: () => void; @@ -319,9 +319,11 @@ const AddAutoRouterTab: React.FC = ({ }; const handleAutomaticSetup = () => { - const matchingPreset = AUTO_SETUP_PRESET_PRIORITY.map((key) => presets.find((preset) => preset.key === key)).find( - (preset) => preset && presetAvailability(preset).kind === "available", - ); + const prioritizedPresets = AUTO_SETUP_PRESET_PRIORITY.flatMap((key) => { + const preset = presets.find((candidate) => candidate.key === key); + return preset ? [preset] : []; + }); + const matchingPreset = prioritizedPresets.find((preset) => presetAvailability(preset).kind === "available"); if (matchingPreset) { const presetState = presetAvailability(matchingPreset); setSelectedPreset(matchingPreset.key); @@ -331,7 +333,8 @@ const AddAutoRouterTab: React.FC = ({ return; } - const generatedConfig = buildAutomaticRouterConfig(modelInfo, deployments ?? [], modelCostMap); + const preferredTierModels = buildPreferredTierModels(prioritizedPresets, availability); + const generatedConfig = buildAutomaticRouterConfig(modelInfo, deployments ?? [], modelCostMap, preferredTierModels); if (generatedConfig === null) { toast.fromError("Add at least one chat model before configuring an Auto Router"); return; diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts index b5340dbad2e..978cdf09217 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; -import { buildAutomaticRouterConfig } from "./auto_setup"; +import { buildAutomaticRouterConfig, type PreferredTierModels } from "./auto_setup"; const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" })); @@ -24,6 +24,68 @@ const tierModels = (config: ReturnType) => ]; describe("buildAutomaticRouterConfig", () => { + it("uses available preferred models before price ranking", () => { + const preferred: PreferredTierModels = { + SIMPLE: ["preferred-simple"], + MEDIUM: ["preferred-medium"], + COMPLEX: ["preferred-complex"], + REASONING: ["preferred-reasoning"], + }; + const available = [...Object.values(preferred).flat(), "cheap-decoy", "expensive-decoy"]; + const config = buildAutomaticRouterConfig( + models(...available), + available.map((name, index) => deployment(name, index + 1)), + {}, + preferred, + ); + + expect(tierModels(config)).toEqual([ + "preferred-simple", + "preferred-medium", + "preferred-complex", + "preferred-reasoning", + ]); + }); + + it("reuses the nearest preferred model for tiers with no preferred match", () => { + const preferred: PreferredTierModels = { + SIMPLE: ["preferred-simple"], + MEDIUM: [], + COMPLEX: ["preferred-complex"], + REASONING: [], + }; + const config = buildAutomaticRouterConfig( + models("preferred-simple", "preferred-complex", "cheap-decoy"), + [deployment("preferred-simple", 4), deployment("preferred-complex", 5), deployment("cheap-decoy", 1)], + {}, + preferred, + ); + + expect(tierModels(config)).toEqual([ + "preferred-simple", + "preferred-simple", + "preferred-complex", + "preferred-complex", + ]); + }); + + it("uses price ranking when none of the preferred models are available", () => { + const unavailablePreferred: PreferredTierModels = { + SIMPLE: ["missing-simple"], + MEDIUM: ["missing-medium"], + COMPLEX: ["missing-complex"], + REASONING: ["missing-reasoning"], + }; + const config = buildAutomaticRouterConfig( + models("expensive", "cheap", "premium", "middle"), + [deployment("cheap", 1), deployment("middle", 2), deployment("premium", 3), deployment("expensive", 4)], + {}, + unavailablePreferred, + ); + + expect(tierModels(config)).toEqual(["cheap", "middle", "premium", "expensive"]); + }); + it("uses four different models when four are available", () => { const config = buildAutomaticRouterConfig( models("expensive", "cheap", "premium", "middle"), diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts index 265a943e859..1540a2cd360 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts @@ -1,5 +1,6 @@ import { isAutoRouterDeployment, type AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import type { ModelGroup } from "@/components/llm_calls/fetch_models"; +import { resolveAvailableModel, type AutoRouterPreset, type ModelAvailability } from "@/lib/autorouter_presets"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; type ModelCost = { @@ -9,6 +10,10 @@ type ModelCost = { export type ModelCostMap = Record; +const TIER_NAMES = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const; +type TierName = (typeof TIER_NAMES)[number]; +export type PreferredTierModels = Record; + const price = (cost: ModelCost | null | undefined): number | undefined => { const input = cost?.input_cost_per_token; const output = cost?.output_cost_per_token; @@ -56,10 +61,46 @@ const selectTierModels = (ranked: string[]): [string, string, string, string] => return [ranked[0], ranked[Math.floor(last / 3)], ranked[Math.floor((2 * last) / 3)], ranked[last]]; }; +export const buildPreferredTierModels = ( + presets: AutoRouterPreset[], + availability: ModelAvailability, +): PreferredTierModels => + Object.fromEntries( + TIER_NAMES.map((tier) => [ + tier, + Array.from( + new Set( + presets.flatMap((preset) => + preset.complexity_router_config.tiers[tier].flatMap((model) => { + const resolved = resolveAvailableModel(model, availability); + return resolved ? [resolved] : []; + }), + ), + ), + ), + ]), + ) as PreferredTierModels; + +const selectPreferredTierModels = ( + preferredByTier: PreferredTierModels, + usableNames: ReadonlySet, +): [string, string, string, string] | null => { + const preferred = TIER_NAMES.map((tier) => preferredByTier[tier].find((name) => usableNames.has(name))); + const candidates = preferred.flatMap((model, tier) => (model ? [{ model, tier }] : [])); + if (candidates.length === 0) return null; + + const nearest = (tier: number): string => + [...candidates].sort( + (left, right) => Math.abs(left.tier - tier) - Math.abs(right.tier - tier) || left.tier - right.tier, + )[0].model; + return preferred.map((model, tier) => model ?? nearest(tier)) as [string, string, string, string]; +}; + export const buildAutomaticRouterConfig = ( models: ModelGroup[], deployments: AutoRouterDeployment[], costMap: ModelCostMap, + preferredByTier?: PreferredTierModels, ): ComplexityRouterConfigValue | null => { const autoRouterNames: ReadonlySet = new Set( deployments @@ -75,6 +116,7 @@ export const buildAutomaticRouterConfig = ( ), ); if (names.length === 0) return null; + const usableNames: ReadonlySet = new Set(names); const ranked = names .map((name) => ({ name, price: groupPrice(name, deployments, costMap) })) @@ -88,7 +130,9 @@ export const buildAutomaticRouterConfig = ( }) .map(({ name }) => name); - const selected = selectTierModels(ranked); + const selected = preferredByTier + ? selectPreferredTierModels(preferredByTier, usableNames) ?? selectTierModels(ranked) + : selectTierModels(ranked); return { tiers: { diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index b01108f1631..2aafbfcfcd9 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -152,7 +152,7 @@ export const deploymentRefsFromModelInfo = ( return row.model_name && underlyingModels.length > 0 ? [{ modelGroup: row.model_name, underlyingModels }] : []; }); -const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => { +export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => { const { modelGroups, underlyingIndex } = availability; if (modelGroups.has(requiredModel)) return requiredModel; const normalized = normalizeModelName(requiredModel);