From 940fdfb26bb9de0e439cc3d021cf79eef570eae8 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 18:36:25 -0700 Subject: [PATCH 01/10] feat(ui): add one-click Auto Router setup --- .../app/(dashboard)/hooks/models/useModels.ts | 5 + .../add_model/add_auto_router_tab.test.tsx | 55 ++++++++++ .../add_model/add_auto_router_tab.tsx | 48 +++++++++ .../components/add_model/auto_setup.test.ts | 101 ++++++++++++++++++ .../src/components/add_model/auto_setup.ts | 95 ++++++++++++++++ 5 files changed, 304 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts create mode 100644 ui/litellm-dashboard/src/components/add_model/auto_setup.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index a9f7c54698a..e18406a9e60 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -90,6 +90,9 @@ export interface AutoRouterCandidateDeployment { export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { litellm_params?: { model?: string | null; + base_model?: string | null; + input_cost_per_token?: number | null; + output_cost_per_token?: number | null; complexity_router_config?: unknown; complexity_router_default_model?: string | null; auto_router_config?: unknown; @@ -105,6 +108,8 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { /** False for config.yaml-defined deployments, which the update and delete routes refuse. */ db_model?: boolean | null; base_model?: string | null; + input_cost_per_token?: number | null; + output_cost_per_token?: number | null; created_at?: string | null; updated_at?: string | null; team_id?: string | null; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index dfef8171c51..8069e7a0504 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -19,6 +19,9 @@ vi.mock( "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets", async () => await import("../../../tests/mocks/autoRouterPresets"), ); +vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ + useModelCostMap: () => ({ data: {}, isLoading: false }), +})); const getAllPresets = (): AutoRouterPreset[] => BUNDLED_PRESETS; const getPresetByKey = (key: string): AutoRouterPreset | undefined => BUNDLED_PRESETS.find((p) => p.key === key); @@ -155,6 +158,58 @@ describe("AddAutoRouterTab", () => { expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument(); }); + it("configures four unique tiers from the available models with one click", async () => { + mockFetchAvailableModels.mockResolvedValue([ + { model_group: "premium", mode: "chat" }, + { model_group: "cheap", mode: "chat" }, + { model_group: "best", mode: "chat" }, + { model_group: "middle", mode: "chat" }, + ]); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "cheap", litellm_params: { model: "cheap", input_cost_per_token: 1 } }, + { model_name: "middle", litellm_params: { model: "middle", input_cost_per_token: 2 } }, + { model_name: "premium", litellm_params: { model: "premium", input_cost_per_token: 3 } }, + { model_name: "best", litellm_params: { model: "best", input_cost_per_token: 4 } }, + ]); + renderWithProviders(); + + const button = screen.getByTestId("configure-automatically-button"); + await waitFor(() => expect(button).toBeEnabled()); + await userEvent.click(button); + + expect(screen.getByText(/Simple: cheap.*Medium: middle.*Complex: premium.*Reasoning: best/)).toBeInTheDocument(); + }); + + it("prefers the first compatible bundled template over the price fallback", async () => { + const firstPreset = getAllPresets()[0]; + mockFetchAvailableModels.mockResolvedValue( + [...getRequiredModelsInPreset(firstPreset)].map((model_group) => ({ model_group, mode: "chat" })), + ); + mockFetchAllModelDeployments.mockResolvedValue([]); + renderWithProviders(); + + const button = screen.getByTestId("configure-automatically-button"); + await waitFor(() => expect(button).toBeEnabled()); + await userEvent.click(button); + + expect(toast.success).toHaveBeenCalledWith(`Configured with ${firstPreset.label}`); + }); + + it("prefers the OpenAI template over Gemini", async () => { + const openAiPreset = getPresetByKey("openai_family")!; + const geminiPreset = getPresetByKey("gemini_family")!; + const available = new Set([...getRequiredModelsInPreset(openAiPreset), ...getRequiredModelsInPreset(geminiPreset)]); + mockFetchAvailableModels.mockResolvedValue([...available].map((model_group) => ({ model_group, mode: "chat" }))); + mockFetchAllModelDeployments.mockResolvedValue([]); + renderWithProviders(); + + const button = screen.getByTestId("configure-automatically-button"); + await waitFor(() => expect(button).toBeEnabled()); + await userEvent.click(button); + + expect(toast.success).toHaveBeenCalledWith(`Configured with ${openAiPreset.label}`); + }); + // Nothing is filled in, so there is nothing to submit. The button reports that itself instead of // accepting a click and answering with a toast. it("offers no submit at all until every tier has a model", async () => { diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 1a1725b8dd0..d671403717b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -21,6 +21,7 @@ import TeamDropdown from "../common_components/team_dropdown"; import { type AddAutoRouterValues, handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; import { fetchAvailableModels, type ModelGroup } from "@/components/llm_calls/fetch_models"; import { autoRouterListKey, fetchAllModelDeployments } from "@/app/(dashboard)/hooks/models/useModels"; +import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import ComplexityRouterConfig, { ComplexityRouterConfigValue, effectiveClassifierType, @@ -64,6 +65,7 @@ import { } from "@/lib/autorouter_presets"; import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { buildAutomaticRouterConfig } from "./auto_setup"; interface AddAutoRouterTabProps { handleOk: () => void; @@ -104,6 +106,7 @@ const presetDisabledHint = (availability: PresetAvailability): string | null => const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models"; const NO_PRESETS: AutoRouterPreset[] = []; +const AUTO_SETUP_PRESET_PRIORITY = ["1m_context", "anthropic_family", "openai_family", "gemini_family", "lite"]; // A one-line summary of what's configured, shown when the detailed section is collapsed so a // caller can see the shape of the config without opening it. @@ -233,6 +236,7 @@ const AddAutoRouterTab: React.FC = ({ queryFn: () => fetchAllModelDeployments(accessToken, userId ?? "", userRole), enabled: Boolean(accessToken), }); + const { data: modelCostMap = {}, isLoading: costsLoading } = useModelCostMap(); const modelsLoading = groupsLoading || deploymentsLoading; const modelInfo = React.useMemo(() => data ?? [], [data]); const { @@ -242,6 +246,7 @@ const AddAutoRouterTab: React.FC = ({ refetch: refetchPresets, } = useAutoRouterPresets(); const presets = presetsData ?? NO_PRESETS; + const automaticSetupLoading = modelsLoading || costsLoading || presetsPending; const presetsUnavailable = presetsError && presetsData === undefined; // react-query keeps the last successful list around when a later refetch fails, so isError alone // can't tell "never loaded" apart from "loaded, then a background refetch errored" - only the @@ -313,6 +318,30 @@ const AddAutoRouterTab: React.FC = ({ setEscalationKeywords(prefill.escalationKeywords); }; + const handleAutomaticSetup = () => { + const matchingPreset = AUTO_SETUP_PRESET_PRIORITY.map((key) => presets.find((preset) => preset.key === key)).find( + (preset) => preset && presetAvailability(preset).kind === "available", + ); + if (matchingPreset) { + const presetState = presetAvailability(matchingPreset); + setSelectedPreset(matchingPreset.key); + applyPrefill(buildPresetPrefill(matchingPreset.complexity_router_config, availability)); + setDetailsExpanded(presetState.kind === "available" && presetState.viaDeployments); + toast.success(`Configured with ${matchingPreset.label}`); + return; + } + + const generatedConfig = buildAutomaticRouterConfig(modelInfo, deployments ?? [], modelCostMap); + if (generatedConfig === null) { + toast.fromError("Add at least one chat model before configuring an Auto Router"); + return; + } + setSelectedPreset(undefined); + applyPrefill({ ...buildEmptyPrefill(), complexityRouterConfig: generatedConfig }); + setDetailsExpanded(false); + toast.success("Automatic setup created", { description: tierConfigSummary(generatedConfig) }); + }; + const handlePresetChange = (presetKey: string | undefined) => { if (!presetKey || presetKey === "custom") { setSelectedPreset(presetKey); @@ -508,6 +537,25 @@ const AddAutoRouterTab: React.FC = ({ {({ ref, ...field }) => } +
+
+
Start with a recommended setup
+
+ Uses your available models and their listed prices. You can review and edit everything before + saving. +
+
+ +
+
} -
-
-
Start with a recommended setup
-
- Uses your available models and our recommended setups. You can review and edit everything before - saving. + {!automaticSetupLoading && automaticRouterConfig && ( +
+
+
Start with a recommended setup
+
+ Uses your available models and our recommended setups. You can review and edit everything before + saving. +
+
- -
+ )}
diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts index 978cdf09217..3722086eaae 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts @@ -1,184 +1,89 @@ import { describe, expect, it } from "vitest"; import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; -import { buildAutomaticRouterConfig, type PreferredTierModels } from "./auto_setup"; +import { buildModelAvailability } from "@/lib/autorouter_presets"; +import { buildAutomaticRouterConfig, buildPreferredTierModels, type PreferredTierModels } from "./auto_setup"; const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" })); +const deployment = (model_name: string, model = model_name): AutoRouterDeployment => ({ + model_name, + litellm_params: { model }, +}); +const tierModels = (config: ReturnType) => + config && Object.values(config.tiers).map((tier) => (typeof tier === "string" ? tier : tier[0])); -const deployment = (name: string, cost: number): AutoRouterDeployment => ({ - model_name: name, - litellm_params: { - model: name, - input_cost_per_token: cost / 2, - output_cost_per_token: cost / 2, - }, +describe("buildPreferredTierModels", () => { + it("recognizes curated models that are not in a preset", () => { + const available = ["gpt-4o-mini", "claude-sonnet-4-5", "grok-4", "deepseek-reasoner"]; + const availability = buildModelAvailability(available, []); + const preferred = buildPreferredTierModels([], availability); + + expect(preferred).toEqual({ + SIMPLE: ["gpt-4o-mini"], + MEDIUM: ["claude-sonnet-4-5"], + COMPLEX: ["grok-4"], + REASONING: ["deepseek-reasoner"], + }); + }); }); -const firstModel = (value: string | string[]): string => (typeof value === "string" ? value : value[0]); - -const tierModels = (config: ReturnType) => - config && [ - firstModel(config.tiers.SIMPLE), - firstModel(config.tiers.MEDIUM), - firstModel(config.tiers.COMPLEX), - firstModel(config.tiers.REASONING), - ]; - describe("buildAutomaticRouterConfig", () => { - it("uses available preferred models before price ranking", () => { + it("selects one preferred model for each tier", () => { const preferred: PreferredTierModels = { - SIMPLE: ["preferred-simple"], - MEDIUM: ["preferred-medium"], - COMPLEX: ["preferred-complex"], - REASONING: ["preferred-reasoning"], + SIMPLE: ["simple"], + MEDIUM: ["medium"], + COMPLEX: ["complex"], + REASONING: ["reasoning"], }; - const available = [...Object.values(preferred).flat(), "cheap-decoy", "expensive-decoy"]; - const config = buildAutomaticRouterConfig( - models(...available), - available.map((name, index) => deployment(name, index + 1)), - {}, - preferred, - ); - expect(tierModels(config)).toEqual([ - "preferred-simple", - "preferred-medium", - "preferred-complex", - "preferred-reasoning", - ]); + expect( + tierModels(buildAutomaticRouterConfig(models("simple", "medium", "complex", "reasoning"), [], preferred)), + ).toEqual(["simple", "medium", "complex", "reasoning"]); }); - it("reuses the nearest preferred model for tiers with no preferred match", () => { + it("reuses the closest available tier when a tier has no match", () => { const preferred: PreferredTierModels = { - SIMPLE: ["preferred-simple"], + SIMPLE: ["simple"], MEDIUM: [], - COMPLEX: ["preferred-complex"], + COMPLEX: ["complex"], REASONING: [], }; - const config = buildAutomaticRouterConfig( - models("preferred-simple", "preferred-complex", "cheap-decoy"), - [deployment("preferred-simple", 4), deployment("preferred-complex", 5), deployment("cheap-decoy", 1)], - {}, - preferred, - ); - expect(tierModels(config)).toEqual([ - "preferred-simple", - "preferred-simple", - "preferred-complex", - "preferred-complex", + expect(tierModels(buildAutomaticRouterConfig(models("simple", "complex"), [], preferred))).toEqual([ + "simple", + "simple", + "complex", + "complex", ]); }); - it("uses price ranking when none of the preferred models are available", () => { - const unavailablePreferred: PreferredTierModels = { + it("returns null when none of the available models are recommended", () => { + const preferred: PreferredTierModels = { SIMPLE: ["missing-simple"], MEDIUM: ["missing-medium"], COMPLEX: ["missing-complex"], REASONING: ["missing-reasoning"], }; - const config = buildAutomaticRouterConfig( - models("expensive", "cheap", "premium", "middle"), - [deployment("cheap", 1), deployment("middle", 2), deployment("premium", 3), deployment("expensive", 4)], - {}, - unavailablePreferred, - ); - expect(tierModels(config)).toEqual(["cheap", "middle", "premium", "expensive"]); + expect(buildAutomaticRouterConfig(models("unknown-model"), [], preferred)).toBeNull(); }); - it("uses four different models when four are available", () => { - const config = buildAutomaticRouterConfig( - models("expensive", "cheap", "premium", "middle"), - [deployment("cheap", 1), deployment("middle", 2), deployment("premium", 3), deployment("expensive", 4)], - {}, - ); - - expect(tierModels(config)).toEqual(["cheap", "middle", "premium", "expensive"]); - expect(config?.classifier_type).toBe("heuristic_v2"); - }); - - it("selects one model per tier from a large inventory", () => { - const names = Array.from({ length: 100 }, (_, index) => `model-${index.toString().padStart(3, "0")}`); - const config = buildAutomaticRouterConfig( - models(...names), - names.map((name, index) => deployment(name, index + 1)), - {}, - ); - const expectedTiers = { - SIMPLE: ["model-000"], - MEDIUM: ["model-033"], - COMPLEX: ["model-066"], - REASONING: ["model-099"], + it("ignores non-chat models and existing auto routers", () => { + const preferred: PreferredTierModels = { + SIMPLE: ["gpt-4o-mini", "smart-router"], + MEDIUM: [], + COMPLEX: [], + REASONING: [], }; + const available = [ + { model_group: "gpt-4o-mini", mode: "chat" }, + { model_group: "image-model", mode: "image_generation" }, + { model_group: "smart-router", mode: "chat" }, + ]; - expect(config?.tiers).toEqual(expectedTiers); - }); - - it("only repeats models when fewer than four are available", () => { expect( tierModels( - buildAutomaticRouterConfig( - models("cheap", "expensive"), - [deployment("cheap", 1), deployment("expensive", 4)], - {}, - ), + buildAutomaticRouterConfig(available, [deployment("smart-router", "auto_router/complexity_router")], preferred), ), - ).toEqual(["cheap", "cheap", "expensive", "expensive"]); - }); - - it("uses the published cost map when deployments do not define prices", () => { - const config = buildAutomaticRouterConfig( - models("premium", "cheap", "middle"), - [ - { model_name: "premium", litellm_params: { model: "provider/premium" } }, - { model_name: "cheap", litellm_params: { model: "provider/cheap" } }, - { model_name: "middle", litellm_params: { model: "provider/middle" } }, - ], - { - "provider/cheap": { input_cost_per_token: 1, output_cost_per_token: 1 }, - "provider/middle": { input_cost_per_token: 2, output_cost_per_token: 2 }, - "provider/premium": { input_cost_per_token: 3, output_cost_per_token: 3 }, - }, - ); - - expect(tierModels(config)).toEqual(["cheap", "middle", "premium", "premium"]); - }); - - it("uses the most expensive deployment when a group has several", () => { - const config = buildAutomaticRouterConfig( - models("variable", "steady", "premium", "top"), - [ - deployment("variable", 1), - deployment("variable", 8), - deployment("steady", 2), - deployment("premium", 3), - deployment("top", 4), - ], - {}, - ); - - expect(tierModels(config)).toEqual(["steady", "premium", "top", "variable"]); - }); - - it("ignores non-chat and existing auto-router models", () => { - const config = buildAutomaticRouterConfig( - [ - { model_group: "chat-model", mode: "chat" }, - { model_group: "image-model", mode: "image_generation" }, - { model_group: "auto_router/existing", mode: "chat" }, - { model_group: "smart-router", mode: "chat" }, - ], - [ - deployment("chat-model", 1), - { model_name: "smart-router", litellm_params: { model: "auto_router/complexity_router" } }, - ], - {}, - ); - - expect(tierModels(config)).toEqual(["chat-model", "chat-model", "chat-model", "chat-model"]); - }); - - it("returns null when there are no usable models", () => { - expect(buildAutomaticRouterConfig([], [], {})).toBeNull(); + ).toEqual(["gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini"]); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts index 1540a2cd360..818b8fe4b49 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts @@ -3,62 +3,15 @@ import type { ModelGroup } from "@/components/llm_calls/fetch_models"; import { resolveAvailableModel, type AutoRouterPreset, type ModelAvailability } from "@/lib/autorouter_presets"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; -type ModelCost = { - input_cost_per_token?: number | null; - output_cost_per_token?: number | null; -}; - -export type ModelCostMap = Record; - const TIER_NAMES = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const; type TierName = (typeof TIER_NAMES)[number]; export type PreferredTierModels = Record; -const price = (cost: ModelCost | null | undefined): number | undefined => { - const input = cost?.input_cost_per_token; - const output = cost?.output_cost_per_token; - if (typeof input !== "number" && typeof output !== "number") return undefined; - return (input ?? 0) + (output ?? 0); -}; - -const deploymentPrice = (deployment: AutoRouterDeployment, costMap: ModelCostMap): number | undefined => { - const configured = price(deployment.litellm_params) ?? price(deployment.model_info); - if (configured !== undefined) return configured; - - const references = [ - deployment.litellm_params?.model, - deployment.litellm_params?.base_model, - deployment.model_info?.base_model, - ]; - for (const reference of references) { - if (reference && costMap[reference]) return price(costMap[reference]); - } - return undefined; -}; - -const groupPrice = ( - modelGroup: string, - deployments: AutoRouterDeployment[], - costMap: ModelCostMap, -): number | undefined => { - const groupDeployments = deployments.filter( - (deployment) => - deployment.model_name === modelGroup && !deployment.litellm_params?.model?.startsWith("auto_router/"), - ); - if (groupDeployments.length === 0) return price(costMap[modelGroup]); - const prices = groupDeployments.map((deployment) => deploymentPrice(deployment, costMap)); - if (prices.some((value) => value === undefined)) return undefined; - const knownPrices = prices.filter((value): value is number => value !== undefined); - return Math.max(...knownPrices); -}; - -const selectTierModels = (ranked: string[]): [string, string, string, string] => { - if (ranked.length === 1) return [ranked[0], ranked[0], ranked[0], ranked[0]]; - if (ranked.length === 2) return [ranked[0], ranked[0], ranked[1], ranked[1]]; - if (ranked.length === 3) return [ranked[0], ranked[1], ranked[2], ranked[2]]; - - const last = ranked.length - 1; - return [ranked[0], ranked[Math.floor(last / 3)], ranked[Math.floor((2 * last) / 3)], ranked[last]]; +const ADDITIONAL_TIER_MODELS: PreferredTierModels = { + SIMPLE: ["gpt-4o-mini", "gpt-5-mini", "gemini-2.5-flash", "deepseek-chat"], + MEDIUM: ["gpt-5-mini", "gpt-4o", "claude-sonnet-4-5", "gemini-2.5-flash", "deepseek-chat"], + COMPLEX: ["gpt-5", "gpt-4o", "claude-sonnet-4-6", "gemini-2.5-pro", "grok-4"], + REASONING: ["o3", "deepseek-reasoner", "claude-opus-4-6", "gemini-2.5-pro", "gpt-5"], }; export const buildPreferredTierModels = ( @@ -70,12 +23,13 @@ export const buildPreferredTierModels = ( tier, Array.from( new Set( - presets.flatMap((preset) => - preset.complexity_router_config.tiers[tier].flatMap((model) => { - const resolved = resolveAvailableModel(model, availability); - return resolved ? [resolved] : []; - }), - ), + [ + ...presets.flatMap((preset) => preset.complexity_router_config.tiers[tier]), + ...ADDITIONAL_TIER_MODELS[tier], + ].flatMap((model) => { + const resolved = resolveAvailableModel(model, availability); + return resolved ? [resolved] : []; + }), ), ), ]), @@ -99,8 +53,7 @@ const selectPreferredTierModels = ( export const buildAutomaticRouterConfig = ( models: ModelGroup[], deployments: AutoRouterDeployment[], - costMap: ModelCostMap, - preferredByTier?: PreferredTierModels, + preferredByTier: PreferredTierModels, ): ComplexityRouterConfigValue | null => { const autoRouterNames: ReadonlySet = new Set( deployments @@ -117,22 +70,8 @@ export const buildAutomaticRouterConfig = ( ); if (names.length === 0) return null; const usableNames: ReadonlySet = new Set(names); - - const ranked = names - .map((name) => ({ name, price: groupPrice(name, deployments, costMap) })) - .sort((left, right) => { - if (left.price === undefined && right.price !== undefined) return 1; - if (left.price !== undefined && right.price === undefined) return -1; - if (left.price !== undefined && right.price !== undefined && left.price !== right.price) { - return left.price - right.price; - } - return left.name.localeCompare(right.name); - }) - .map(({ name }) => name); - - const selected = preferredByTier - ? selectPreferredTierModels(preferredByTier, usableNames) ?? selectTierModels(ranked) - : selectTierModels(ranked); + const selected = selectPreferredTierModels(preferredByTier, usableNames); + if (selected === null) return null; return { tiers: { From 78c40ed6a760ca4cd2b352866390381c8fd0d64d Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 19:30:04 -0700 Subject: [PATCH 06/10] style(ui): compact Auto Setup control --- .../add_model/add_auto_router_tab.tsx | 17 ++++++++--------- .../src/components/add_model/auto_setup.test.ts | 7 ++++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 2dfb8864faa..f1f6cbd3246 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -527,15 +527,14 @@ const AddAutoRouterTab: React.FC = ({ {!automaticSetupLoading && automaticRouterConfig && ( -
-
-
Start with a recommended setup
-
- Uses your available models and our recommended setups. You can review and edit everything before - saving. -
-
-
diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts index 3722086eaae..2b63103ee3a 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts @@ -16,13 +16,14 @@ describe("buildPreferredTierModels", () => { const available = ["gpt-4o-mini", "claude-sonnet-4-5", "grok-4", "deepseek-reasoner"]; const availability = buildModelAvailability(available, []); const preferred = buildPreferredTierModels([], availability); - - expect(preferred).toEqual({ + const expected: PreferredTierModels = { SIMPLE: ["gpt-4o-mini"], MEDIUM: ["claude-sonnet-4-5"], COMPLEX: ["grok-4"], REASONING: ["deepseek-reasoner"], - }); + }; + + expect(preferred).toEqual(expected); }); }); From 2cb27985d9b8c74e9dfac49d05f9f26f9e4beece Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 19:43:55 -0700 Subject: [PATCH 07/10] fix(ui): refresh Auto Setup model ladders --- .../add_model/add_auto_router_tab.tsx | 4 +- .../components/add_model/auto_setup.test.ts | 73 +++++++++++++++++-- .../src/components/add_model/auto_setup.ts | 31 ++++++-- 3 files changed, 95 insertions(+), 13 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index f1f6cbd3246..2166114703c 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -273,8 +273,8 @@ const AddAutoRouterTab: React.FC = ({ [presets, availability], ); const automaticRouterConfig = React.useMemo( - () => buildAutomaticRouterConfig(modelInfo, deployments ?? [], preferredTierModels), - [modelInfo, deployments, preferredTierModels], + () => buildAutomaticRouterConfig(modelInfo, deployments ?? [], preferredTierModels, availability), + [modelInfo, deployments, preferredTierModels, availability], ); // A preset's models can only be trusted against a successfully loaded list. Selection and the diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts index 2b63103ee3a..b158b7f3b6e 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts @@ -13,18 +13,38 @@ const tierModels = (config: ReturnType) => describe("buildPreferredTierModels", () => { it("recognizes curated models that are not in a preset", () => { - const available = ["gpt-4o-mini", "claude-sonnet-4-5", "grok-4", "deepseek-reasoner"]; + const available = ["gpt-5.6-luna", "claude-sonnet-5", "grok-4.6", "deepseek-v4-pro"]; const availability = buildModelAvailability(available, []); const preferred = buildPreferredTierModels([], availability); const expected: PreferredTierModels = { - SIMPLE: ["gpt-4o-mini"], - MEDIUM: ["claude-sonnet-4-5"], - COMPLEX: ["grok-4"], - REASONING: ["deepseek-reasoner"], + SIMPLE: ["gpt-5.6-luna"], + MEDIUM: ["claude-sonnet-5"], + COMPLEX: ["deepseek-v4-pro", "grok-4.6"], + REASONING: ["deepseek-v4-pro", "grok-4.6"], }; expect(preferred).toEqual(expected); }); + + it("prefers the current model ladder over older preset entries", () => { + const availability = buildModelAvailability(["gpt-5.6-luna", "gpt-4o-mini"], []); + const preferred = buildPreferredTierModels( + [ + { + key: "old", + label: "Old", + description: "Old model", + complexity_router_config: { + tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "heuristic_v2", + }, + }, + ], + availability, + ); + + expect(preferred.SIMPLE).toEqual(["gpt-5.6-luna", "gpt-4o-mini"]); + }); }); describe("buildAutomaticRouterConfig", () => { @@ -41,6 +61,49 @@ describe("buildAutomaticRouterConfig", () => { ).toEqual(["simple", "medium", "complex", "reasoning"]); }); + it.each([ + { + provider: "OpenAI", + available: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra"], + expected: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra", "gpt-6-astra"], + effort: "max", + }, + { + provider: "Anthropic", + available: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"], + expected: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5", "claude-opus-5"], + effort: "max", + }, + { + provider: "Google", + available: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview"], + expected: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3.1-pro-preview"], + effort: "high", + }, + { + provider: "DeepSeek", + available: ["deepseek-v4-flash", "deepseek-v4-pro"], + expected: ["deepseek-v4-flash", "deepseek-v4-flash", "deepseek-v4-pro", "deepseek-v4-pro"], + effort: "high", + }, + { + provider: "xAI", + available: ["grok-4.6"], + expected: ["grok-4.6", "grok-4.6", "grok-4.6", "grok-4.6"], + effort: "xhigh", + }, + ])("uses the current $provider ladder and maximum reasoning effort", ({ available, expected, effort }) => { + const availability = buildModelAvailability(available, []); + const preferred = buildPreferredTierModels([], availability); + + const config = buildAutomaticRouterConfig(models(...available), [], preferred, availability); + + expect(tierModels(config)).toEqual(expected); + expect(config?.tier_model_params).toEqual({ + REASONING: { [expected[3]]: { reasoning_effort: effort } }, + }); + }); + it("reuses the closest available tier when a tier has no match", () => { const preferred: PreferredTierModels = { SIMPLE: ["simple"], diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts index 818b8fe4b49..0e8ef7c9824 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts @@ -7,11 +7,20 @@ const TIER_NAMES = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const; type TierName = (typeof TIER_NAMES)[number]; export type PreferredTierModels = Record; -const ADDITIONAL_TIER_MODELS: PreferredTierModels = { - SIMPLE: ["gpt-4o-mini", "gpt-5-mini", "gemini-2.5-flash", "deepseek-chat"], - MEDIUM: ["gpt-5-mini", "gpt-4o", "claude-sonnet-4-5", "gemini-2.5-flash", "deepseek-chat"], - COMPLEX: ["gpt-5", "gpt-4o", "claude-sonnet-4-6", "gemini-2.5-pro", "grok-4"], - REASONING: ["o3", "deepseek-reasoner", "claude-opus-4-6", "gemini-2.5-pro", "gpt-5"], +const CURRENT_TIER_MODELS: PreferredTierModels = { + SIMPLE: ["gpt-5.6-luna", "claude-haiku-4-5", "gemini-3.5-flash-lite", "deepseek-v4-flash"], + MEDIUM: ["gpt-5.6-terra", "claude-sonnet-5", "gemini-3.8-flash", "deepseek-v4-flash"], + COMPLEX: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"], + REASONING: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"], +}; + +const MAX_REASONING_EFFORT: Record = { + "gpt-6-astra": "max", + "gpt-5.6-sol": "max", + "claude-opus-5": "max", + "gemini-3.1-pro-preview": "high", + "deepseek-v4-pro": "high", + "grok-4.6": "xhigh", }; export const buildPreferredTierModels = ( @@ -24,8 +33,8 @@ export const buildPreferredTierModels = ( Array.from( new Set( [ + ...CURRENT_TIER_MODELS[tier], ...presets.flatMap((preset) => preset.complexity_router_config.tiers[tier]), - ...ADDITIONAL_TIER_MODELS[tier], ].flatMap((model) => { const resolved = resolveAvailableModel(model, availability); return resolved ? [resolved] : []; @@ -54,6 +63,7 @@ export const buildAutomaticRouterConfig = ( models: ModelGroup[], deployments: AutoRouterDeployment[], preferredByTier: PreferredTierModels, + availability?: ModelAvailability, ): ComplexityRouterConfigValue | null => { const autoRouterNames: ReadonlySet = new Set( deployments @@ -73,6 +83,12 @@ export const buildAutomaticRouterConfig = ( const selected = selectPreferredTierModels(preferredByTier, usableNames); if (selected === null) return null; + const reasoningEffort = + availability && + Object.entries(MAX_REASONING_EFFORT).find( + ([model]) => resolveAvailableModel(model, availability) === selected[3], + )?.[1]; + return { tiers: { SIMPLE: [selected[0]], @@ -81,5 +97,8 @@ export const buildAutomaticRouterConfig = ( REASONING: [selected[3]], }, classifier_type: "heuristic_v2", + ...(reasoningEffort && { + tier_model_params: { REASONING: { [selected[3]]: { reasoning_effort: reasoningEffort } } }, + }), }; }; From 7e2f345a8691eb867bd36b5e0493ad821c7889c3 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 19:46:07 -0700 Subject: [PATCH 08/10] style(ui): place Auto Setup under templates --- .../add_model/add_auto_router_tab.tsx | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 2166114703c..38c44f1d441 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -526,20 +526,6 @@ const AddAutoRouterTab: React.FC = ({ {({ ref, ...field }) => } - {!automaticSetupLoading && automaticRouterConfig && ( -
- -
- )} -
+ {!automaticSetupLoading && automaticRouterConfig && ( + + )} {modelsUnverifiable && (
Could not load available models.{" "} From 901e312b17a1c11dd4d1cb76ae8e75c7ccc7d1b0 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 19:47:44 -0700 Subject: [PATCH 09/10] style(ui): position Auto Setup before templates --- .../add_model/add_auto_router_tab.tsx | 143 +++++++++--------- 1 file changed, 74 insertions(+), 69 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 38c44f1d441..917c775760e 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -518,87 +518,92 @@ const AddAutoRouterTab: React.FC = ({
handleAutoRouterSubmit())} noValidate> - - {({ ref, ...field }) => } - -
- - + )} + - return ( - -
-
{preset.label}
-
{preset.description}
- {disabledHint &&
{disabledHint}
} - {matchedHint &&
{matchedHint}
} -
- - ); - })} - -
-
Custom Configuration
-
Define your auto router from scratch
-
- - - {!automaticSetupLoading && automaticRouterConfig && ( )} - {modelsUnverifiable && ( -
- Could not load available models.{" "} - -
- )} - {presetsPending &&
Loading templates...
} - {presetsUnavailable && ( -
- Could not load templates, so only Custom Configuration is shown.{" "} - -
- )} + +
+ + + {modelsUnverifiable && ( +
+ Could not load available models.{" "} + +
+ )} + {presetsPending &&
Loading templates...
} + {presetsUnavailable && ( +
+ Could not load templates, so only Custom Configuration is shown.{" "} + +
+ )} +
{requiresTeamScope && ( From d5481ca0370141442539231f6e2240a58a080516 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 3 Sep 2026 20:01:44 -0700 Subject: [PATCH 10/10] fix(ui): respect reported reasoning efforts --- .../add_model/add_auto_router_tab.tsx | 4 +- .../components/add_model/auto_setup.test.ts | 55 +++++++++++++++++-- .../src/components/add_model/auto_setup.ts | 21 ++----- 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 917c775760e..a548c2c6533 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -273,8 +273,8 @@ const AddAutoRouterTab: React.FC = ({ [presets, availability], ); const automaticRouterConfig = React.useMemo( - () => buildAutomaticRouterConfig(modelInfo, deployments ?? [], preferredTierModels, availability), - [modelInfo, deployments, preferredTierModels, availability], + () => buildAutomaticRouterConfig(modelInfo, deployments ?? [], preferredTierModels), + [modelInfo, deployments, preferredTierModels], ); // A preset's models can only be trusted against a successfully loaded list. Selection and the diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts index b158b7f3b6e..c5784db4501 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts @@ -4,6 +4,12 @@ import { buildModelAvailability } from "@/lib/autorouter_presets"; import { buildAutomaticRouterConfig, buildPreferredTierModels, type PreferredTierModels } from "./auto_setup"; const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" })); +const reasoningModel = (model_group: string, supported_reasoning_efforts: string[]) => ({ + model_group, + mode: "chat", + supports_reasoning: true, + supported_reasoning_efforts, +}); const deployment = (model_name: string, model = model_name): AutoRouterDeployment => ({ model_name, litellm_params: { model }, @@ -66,42 +72,79 @@ describe("buildAutomaticRouterConfig", () => { provider: "OpenAI", available: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra"], expected: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra", "gpt-6-astra"], + supportedEfforts: ["low", "medium", "high", "xhigh", "max"], effort: "max", }, { provider: "Anthropic", available: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"], expected: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5", "claude-opus-5"], + supportedEfforts: ["low", "medium", "high", "max"], effort: "max", }, { provider: "Google", available: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview"], expected: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3.1-pro-preview"], + supportedEfforts: ["low", "medium", "high"], effort: "high", }, { provider: "DeepSeek", available: ["deepseek-v4-flash", "deepseek-v4-pro"], expected: ["deepseek-v4-flash", "deepseek-v4-flash", "deepseek-v4-pro", "deepseek-v4-pro"], + supportedEfforts: ["none", "high"], effort: "high", }, { provider: "xAI", available: ["grok-4.6"], expected: ["grok-4.6", "grok-4.6", "grok-4.6", "grok-4.6"], + supportedEfforts: ["low", "medium", "high", "xhigh"], effort: "xhigh", }, - ])("uses the current $provider ladder and maximum reasoning effort", ({ available, expected, effort }) => { + ])( + "uses the current $provider ladder and strongest advertised reasoning effort", + ({ available, expected, supportedEfforts, effort }) => { + const availability = buildModelAvailability(available, []); + const preferred = buildPreferredTierModels([], availability); + const modelInfo = models(...available).map((model) => + model.model_group === expected[3] ? reasoningModel(model.model_group, supportedEfforts) : model, + ); + + const config = buildAutomaticRouterConfig(modelInfo, [], preferred); + + expect(tierModels(config)).toEqual(expected); + expect(config?.tier_model_params).toEqual({ + REASONING: { [expected[3]]: { reasoning_effort: effort } }, + }); + }, + ); + + it("never exceeds the selected model group's advertised reasoning efforts", () => { + const available = ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol"]; + const availability = buildModelAvailability(available, []); + const preferred = buildPreferredTierModels([], availability); + const modelInfo = [ + ...models("gpt-5.6-luna", "gpt-5.6-terra"), + reasoningModel("gpt-5.6-sol", ["none", "low", "medium", "high", "xhigh"]), + ]; + + const config = buildAutomaticRouterConfig(modelInfo, [], preferred); + + expect(config?.tier_model_params).toEqual({ + REASONING: { "gpt-5.6-sol": { reasoning_effort: "xhigh" } }, + }); + }); + + it("leaves reasoning effort unset when the proxy does not report supported values", () => { + const available = ["grok-4.6"]; const availability = buildModelAvailability(available, []); const preferred = buildPreferredTierModels([], availability); - const config = buildAutomaticRouterConfig(models(...available), [], preferred, availability); + const config = buildAutomaticRouterConfig(models(...available), [], preferred); - expect(tierModels(config)).toEqual(expected); - expect(config?.tier_model_params).toEqual({ - REASONING: { [expected[3]]: { reasoning_effort: effort } }, - }); + expect(config?.tier_model_params).toBeUndefined(); }); it("reuses the closest available tier when a tier has no match", () => { diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts index 0e8ef7c9824..cd7d587e2c2 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts @@ -7,6 +7,8 @@ const TIER_NAMES = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const; type TierName = (typeof TIER_NAMES)[number]; export type PreferredTierModels = Record; +const REASONING_EFFORT_STRENGTH = ["max", "xhigh", "high", "medium", "low", "minimal", "none"] as const; + const CURRENT_TIER_MODELS: PreferredTierModels = { SIMPLE: ["gpt-5.6-luna", "claude-haiku-4-5", "gemini-3.5-flash-lite", "deepseek-v4-flash"], MEDIUM: ["gpt-5.6-terra", "claude-sonnet-5", "gemini-3.8-flash", "deepseek-v4-flash"], @@ -14,15 +16,6 @@ const CURRENT_TIER_MODELS: PreferredTierModels = { REASONING: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"], }; -const MAX_REASONING_EFFORT: Record = { - "gpt-6-astra": "max", - "gpt-5.6-sol": "max", - "claude-opus-5": "max", - "gemini-3.1-pro-preview": "high", - "deepseek-v4-pro": "high", - "grok-4.6": "xhigh", -}; - export const buildPreferredTierModels = ( presets: AutoRouterPreset[], availability: ModelAvailability, @@ -63,7 +56,6 @@ export const buildAutomaticRouterConfig = ( models: ModelGroup[], deployments: AutoRouterDeployment[], preferredByTier: PreferredTierModels, - availability?: ModelAvailability, ): ComplexityRouterConfigValue | null => { const autoRouterNames: ReadonlySet = new Set( deployments @@ -83,11 +75,10 @@ export const buildAutomaticRouterConfig = ( const selected = selectPreferredTierModels(preferredByTier, usableNames); if (selected === null) return null; - const reasoningEffort = - availability && - Object.entries(MAX_REASONING_EFFORT).find( - ([model]) => resolveAvailableModel(model, availability) === selected[3], - )?.[1]; + const supportedReasoningEfforts = models.find( + (model) => model.model_group === selected[3], + )?.supported_reasoning_efforts; + const reasoningEffort = REASONING_EFFORT_STRENGTH.find((effort) => supportedReasoningEfforts?.includes(effort)); return { tiers: {