diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index a9f7c54698a..e18406a9e60 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -90,6 +90,9 @@ export interface AutoRouterCandidateDeployment { export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { litellm_params?: { model?: string | null; + base_model?: string | null; + input_cost_per_token?: number | null; + output_cost_per_token?: number | null; complexity_router_config?: unknown; complexity_router_default_model?: string | null; auto_router_config?: unknown; @@ -105,6 +108,8 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { /** False for config.yaml-defined deployments, which the update and delete routes refuse. */ db_model?: boolean | null; base_model?: string | null; + input_cost_per_token?: number | null; + output_cost_per_token?: number | null; created_at?: string | null; updated_at?: string | null; team_id?: string | null; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index dfef8171c51..8069e7a0504 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -19,6 +19,9 @@ vi.mock( "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets", async () => await import("../../../tests/mocks/autoRouterPresets"), ); +vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ + useModelCostMap: () => ({ data: {}, isLoading: false }), +})); const getAllPresets = (): AutoRouterPreset[] => BUNDLED_PRESETS; const getPresetByKey = (key: string): AutoRouterPreset | undefined => BUNDLED_PRESETS.find((p) => p.key === key); @@ -155,6 +158,58 @@ describe("AddAutoRouterTab", () => { expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); }); + it("configures four unique tiers from the available models with one click", async () => { + mockFetchAvailableModels.mockResolvedValue([ + { model_group: "premium", mode: "chat" }, + { model_group: "cheap", mode: "chat" }, + { model_group: "best", mode: "chat" }, + { model_group: "middle", mode: "chat" }, + ]); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "cheap", litellm_params: { model: "cheap", input_cost_per_token: 1 } }, + { model_name: "middle", litellm_params: { model: "middle", input_cost_per_token: 2 } }, + { model_name: "premium", litellm_params: { model: "premium", input_cost_per_token: 3 } }, + { model_name: "best", litellm_params: { model: "best", input_cost_per_token: 4 } }, + ]); + renderWithProviders(); + + const button = screen.getByTestId("configure-automatically-button"); + await waitFor(() => expect(button).toBeEnabled()); + await userEvent.click(button); + + expect(screen.getByText(/Simple: cheap.*Medium: middle.*Complex: premium.*Reasoning: best/)).toBeInTheDocument(); + }); + + it("prefers the first compatible bundled template over the price fallback", async () => { + const firstPreset = getAllPresets()[0]; + mockFetchAvailableModels.mockResolvedValue( + [...getRequiredModelsInPreset(firstPreset)].map((model_group) => ({ model_group, mode: "chat" })), + ); + mockFetchAllModelDeployments.mockResolvedValue([]); + renderWithProviders(); + + const button = screen.getByTestId("configure-automatically-button"); + await waitFor(() => expect(button).toBeEnabled()); + await userEvent.click(button); + + expect(toast.success).toHaveBeenCalledWith(`Configured with ${firstPreset.label}`); + }); + + it("prefers the OpenAI template over Gemini", async () => { + const openAiPreset = getPresetByKey("openai_family")!; + const geminiPreset = getPresetByKey("gemini_family")!; + const available = new Set([...getRequiredModelsInPreset(openAiPreset), ...getRequiredModelsInPreset(geminiPreset)]); + mockFetchAvailableModels.mockResolvedValue([...available].map((model_group) => ({ model_group, mode: "chat" }))); + mockFetchAllModelDeployments.mockResolvedValue([]); + renderWithProviders(); + + const button = screen.getByTestId("configure-automatically-button"); + await waitFor(() => expect(button).toBeEnabled()); + await userEvent.click(button); + + expect(toast.success).toHaveBeenCalledWith(`Configured with ${openAiPreset.label}`); + }); + // Nothing is filled in, so there is nothing to submit. The button reports that itself instead of // accepting a click and answering with a toast. it("offers no submit at all until every tier has a model", async () => { diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 1a1725b8dd0..d671403717b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -21,6 +21,7 @@ import TeamDropdown from "../common_components/team_dropdown"; import { type AddAutoRouterValues, handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; import { fetchAvailableModels, type ModelGroup } from "@/components/llm_calls/fetch_models"; import { autoRouterListKey, fetchAllModelDeployments } from "@/app/(dashboard)/hooks/models/useModels"; +import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import ComplexityRouterConfig, { ComplexityRouterConfigValue, effectiveClassifierType, @@ -64,6 +65,7 @@ import { } from "@/lib/autorouter_presets"; import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { buildAutomaticRouterConfig } from "./auto_setup"; interface AddAutoRouterTabProps { handleOk: () => void; @@ -104,6 +106,7 @@ const presetDisabledHint = (availability: PresetAvailability): string | null => const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models"; const NO_PRESETS: AutoRouterPreset[] = []; +const AUTO_SETUP_PRESET_PRIORITY = ["1m_context", "anthropic_family", "openai_family", "gemini_family", "lite"]; // A one-line summary of what's configured, shown when the detailed section is collapsed so a // caller can see the shape of the config without opening it. @@ -233,6 +236,7 @@ const AddAutoRouterTab: React.FC = ({ queryFn: () => fetchAllModelDeployments(accessToken, userId ?? "", userRole), enabled: Boolean(accessToken), }); + const { data: modelCostMap = {}, isLoading: costsLoading } = useModelCostMap(); const modelsLoading = groupsLoading || deploymentsLoading; const modelInfo = React.useMemo(() => data ?? [], [data]); const { @@ -242,6 +246,7 @@ const AddAutoRouterTab: React.FC = ({ refetch: refetchPresets, } = useAutoRouterPresets(); const presets = presetsData ?? NO_PRESETS; + const automaticSetupLoading = modelsLoading || costsLoading || presetsPending; const presetsUnavailable = presetsError && presetsData === undefined; // react-query keeps the last successful list around when a later refetch fails, so isError alone // can't tell "never loaded" apart from "loaded, then a background refetch errored" - only the @@ -313,6 +318,30 @@ const AddAutoRouterTab: React.FC = ({ setEscalationKeywords(prefill.escalationKeywords); }; + const handleAutomaticSetup = () => { + const matchingPreset = AUTO_SETUP_PRESET_PRIORITY.map((key) => presets.find((preset) => preset.key === key)).find( + (preset) => preset && presetAvailability(preset).kind === "available", + ); + if (matchingPreset) { + const presetState = presetAvailability(matchingPreset); + setSelectedPreset(matchingPreset.key); + applyPrefill(buildPresetPrefill(matchingPreset.complexity_router_config, availability)); + setDetailsExpanded(presetState.kind === "available" && presetState.viaDeployments); + toast.success(`Configured with ${matchingPreset.label}`); + return; + } + + const generatedConfig = buildAutomaticRouterConfig(modelInfo, deployments ?? [], modelCostMap); + if (generatedConfig === null) { + toast.fromError("Add at least one chat model before configuring an Auto Router"); + return; + } + setSelectedPreset(undefined); + applyPrefill({ ...buildEmptyPrefill(), complexityRouterConfig: generatedConfig }); + setDetailsExpanded(false); + toast.success("Automatic setup created", { description: tierConfigSummary(generatedConfig) }); + }; + const handlePresetChange = (presetKey: string | undefined) => { if (!presetKey || presetKey === "custom") { setSelectedPreset(presetKey); @@ -508,6 +537,25 @@ const AddAutoRouterTab: React.FC = ({ {({ ref, ...field }) => } + + + Start with a recommended setup + + Uses your available models and their listed prices. You can review and edit everything before + saving. + + + + {automaticSetupLoading && } + Configure automatically + + + Template names.map((model_group) => ({ model_group, mode: "chat" })); + +const deployment = (name: string, cost: number): AutoRouterDeployment => ({ + model_name: name, + litellm_params: { + model: name, + input_cost_per_token: cost / 2, + output_cost_per_token: cost / 2, + }, +}); + +const firstModel = (value: string | string[]): string => (typeof value === "string" ? value : value[0]); + +const tierModels = (config: ReturnType) => + config && [ + firstModel(config.tiers.SIMPLE), + firstModel(config.tiers.MEDIUM), + firstModel(config.tiers.COMPLEX), + firstModel(config.tiers.REASONING), + ]; + +describe("buildAutomaticRouterConfig", () => { + it("uses four different models when four are available", () => { + const config = buildAutomaticRouterConfig( + models("expensive", "cheap", "premium", "middle"), + [deployment("cheap", 1), deployment("middle", 2), deployment("premium", 3), deployment("expensive", 4)], + {}, + ); + + expect(tierModels(config)).toEqual(["cheap", "middle", "premium", "expensive"]); + expect(config?.classifier_type).toBe("heuristic_v2"); + }); + + it("only repeats models when fewer than four are available", () => { + expect( + tierModels( + buildAutomaticRouterConfig( + models("cheap", "expensive"), + [deployment("cheap", 1), deployment("expensive", 4)], + {}, + ), + ), + ).toEqual(["cheap", "cheap", "expensive", "expensive"]); + }); + + it("uses the published cost map when deployments do not define prices", () => { + const config = buildAutomaticRouterConfig( + models("premium", "cheap", "middle"), + [ + { model_name: "premium", litellm_params: { model: "provider/premium" } }, + { model_name: "cheap", litellm_params: { model: "provider/cheap" } }, + { model_name: "middle", litellm_params: { model: "provider/middle" } }, + ], + { + "provider/cheap": { input_cost_per_token: 1, output_cost_per_token: 1 }, + "provider/middle": { input_cost_per_token: 2, output_cost_per_token: 2 }, + "provider/premium": { input_cost_per_token: 3, output_cost_per_token: 3 }, + }, + ); + + expect(tierModels(config)).toEqual(["cheap", "middle", "premium", "premium"]); + }); + + it("uses the most expensive deployment when a group has several", () => { + const config = buildAutomaticRouterConfig( + models("variable", "steady", "premium", "top"), + [ + deployment("variable", 1), + deployment("variable", 8), + deployment("steady", 2), + deployment("premium", 3), + deployment("top", 4), + ], + {}, + ); + + expect(tierModels(config)).toEqual(["steady", "premium", "top", "variable"]); + }); + + it("ignores non-chat and existing auto-router models", () => { + const config = buildAutomaticRouterConfig( + [ + { model_group: "chat-model", mode: "chat" }, + { model_group: "image-model", mode: "image_generation" }, + { model_group: "auto_router/existing", mode: "chat" }, + ], + [deployment("chat-model", 1)], + {}, + ); + + expect(tierModels(config)).toEqual(["chat-model", "chat-model", "chat-model", "chat-model"]); + }); + + it("returns null when there are no usable models", () => { + expect(buildAutomaticRouterConfig([], [], {})).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts new file mode 100644 index 00000000000..f5c35ef5150 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts @@ -0,0 +1,95 @@ +import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; +import type { ModelGroup } from "@/components/llm_calls/fetch_models"; +import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; + +type ModelCost = { + input_cost_per_token?: number | null; + output_cost_per_token?: number | null; +}; + +export type ModelCostMap = Record; + +const price = (cost: ModelCost | null | undefined): number | undefined => { + const input = cost?.input_cost_per_token; + const output = cost?.output_cost_per_token; + if (typeof input !== "number" && typeof output !== "number") return undefined; + return (input ?? 0) + (output ?? 0); +}; + +const deploymentPrice = (deployment: AutoRouterDeployment, costMap: ModelCostMap): number | undefined => { + const configured = price(deployment.litellm_params) ?? price(deployment.model_info); + if (configured !== undefined) return configured; + + const references = [ + deployment.litellm_params?.model, + deployment.litellm_params?.base_model, + deployment.model_info?.base_model, + ]; + for (const reference of references) { + if (reference && costMap[reference]) return price(costMap[reference]); + } + return undefined; +}; + +const groupPrice = ( + modelGroup: string, + deployments: AutoRouterDeployment[], + costMap: ModelCostMap, +): number | undefined => { + const groupDeployments = deployments.filter( + (deployment) => + deployment.model_name === modelGroup && !deployment.litellm_params?.model?.startsWith("auto_router/"), + ); + if (groupDeployments.length === 0) return price(costMap[modelGroup]); + const prices = groupDeployments.map((deployment) => deploymentPrice(deployment, costMap)); + if (prices.some((value) => value === undefined)) return undefined; + const knownPrices = prices.filter((value): value is number => value !== undefined); + return Math.max(...knownPrices); +}; + +export const buildAutomaticRouterConfig = ( + models: ModelGroup[], + deployments: AutoRouterDeployment[], + costMap: ModelCostMap, +): ComplexityRouterConfigValue | null => { + const names = Array.from( + new Set( + models + .filter((model) => model.mode === undefined || model.mode === "chat") + .map((model) => model.model_group) + .filter((name) => name && !name.startsWith("auto_router/")), + ), + ); + if (names.length === 0) return null; + + const ranked = names + .map((name) => ({ name, price: groupPrice(name, deployments, costMap) })) + .sort((left, right) => { + if (left.price === undefined && right.price !== undefined) return 1; + if (left.price !== undefined && right.price === undefined) return -1; + if (left.price !== undefined && right.price !== undefined && left.price !== right.price) { + return left.price - right.price; + } + return left.name.localeCompare(right.name); + }) + .map(({ name }) => name); + + let selected: [string, string, string, string]; + if (ranked.length === 1) selected = [ranked[0], ranked[0], ranked[0], ranked[0]]; + else if (ranked.length === 2) selected = [ranked[0], ranked[0], ranked[1], ranked[1]]; + else if (ranked.length === 3) selected = [ranked[0], ranked[1], ranked[2], ranked[2]]; + else { + const last = ranked.length - 1; + selected = [ranked[0], ranked[Math.floor(last / 3)], ranked[Math.floor((2 * last) / 3)], ranked[last]]; + } + + return { + tiers: { + SIMPLE: [selected[0]], + MEDIUM: [selected[1]], + COMPLEX: [selected[2]], + REASONING: [selected[3]], + }, + classifier_type: "heuristic_v2", + }; +};