diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index 24f76065f34..744db87ad11 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -121,6 +121,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { /** False for config.yaml-defined deployments, which the update and delete routes refuse. */ db_model?: boolean | null; base_model?: string | null; + blocked?: boolean | null; mode?: string | null; created_at?: string | null; updated_at?: string | null; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx index 6f7e04f92c6..2d1c24685f0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx @@ -106,10 +106,7 @@ export function AutoRoutersPanel({ > Add Auto Router - - Choose a classifier to route each request to a model. Called like any other model, so clients keep using a - single model name. - + Choose models for your auto router. { + switch (availability.kind) { + case "available": + return null; + case "loading": + return "Checking model availability..."; + case "unverifiable": + return "Cannot verify these models are available"; + case "missing_models": + return `Missing: ${availability.models.join(", ")}`; + } +}; + +// "loading"/"unverifiable" are transient system states, not a gap specific to this preset; only a +// caller-specific missing-model reason gets the alarming red treatment. +const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models"; + +type Props = { + selectedPreset?: string; + presets: AutoRouterPreset[]; + presetAvailability: (preset: AutoRouterPreset) => PresetAvailability; + onPresetChange: (key: string | undefined) => void; + pending: boolean; + unavailable: boolean; + onRetry: () => void; +}; + +export default function AutoRouterPresetSelect({ + selectedPreset, + presets, + presetAvailability, + onPresetChange, + pending, + unavailable, + onRetry, +}: Props) { + const sortedPresetOptions = React.useMemo( + () => + presets + .map((preset) => ({ preset, availability: presetAvailability(preset) })) + .sort((a, b) => Number(b.availability.kind === "available") - Number(a.availability.kind === "available")), + [presets, presetAvailability], + ); + + const templateItems = React.useMemo( + () => [ + ...sortedPresetOptions.map(({ preset }) => ({ value: preset.key, label: preset.label })), + { value: "custom", label: "Start from scratch" }, + ], + [sortedPresetOptions], + ); + + return ( +
+
+ +

Replaces classifier and model choices

+ {pending &&
Loading templates...
} + {unavailable && ( +
+ Could not load templates, so only Start from scratch is shown.{" "} + +
+ )} +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx new file mode 100644 index 00000000000..64c3d4de63f --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterSimpleSetup.tsx @@ -0,0 +1,190 @@ +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; +import { ROUTER_FAMILIES, ROUTER_GOALS, type FamilyPreference, type RouterGoal, type SolverModel } from "./auto_setup"; + +type Props = { + customized: boolean; + hasSuggestions: boolean; + onReset: () => void; + goal: RouterGoal; + family: FamilyPreference; + models: readonly SolverModel[]; + everyday?: SolverModel; + hard?: SolverModel; + waitingForTeam: boolean; + loading: boolean; + error: boolean; + notice: string | null; + onGoalChange: (goal: RouterGoal) => void; + onFamilyChange: (family: FamilyPreference) => void; + onModelChange: (role: "everyday" | "hard", model: string) => void; + onAdvanced: () => void; + onRetry: () => void; +}; + +const sharedFamily = (everyday?: SolverModel, hard?: SolverModel) => + everyday?.family && everyday.family === hard?.family ? ROUTER_FAMILIES[everyday.family] : null; + +const modelDescription = (model: SolverModel) => + [model.model !== model.name ? model.model : null, model.provider].filter(Boolean).join(" · "); + +export default function AutoRouterSimpleSetup({ + customized, + hasSuggestions, + onReset, + goal, + family, + models, + everyday, + hard, + waitingForTeam, + loading, + error, + notice, + onGoalChange, + onFamilyChange, + onModelChange, + onAdvanced, + onRetry, +}: Props) { + const inventoryReady = !waitingForTeam && !loading; + const showPreferences = !inventoryReady || error || hasSuggestions; + const sameFamily = sharedFamily(everyday, hard); + const familyItems = [ + { value: "auto", label: sameFamily ? `Auto (${sameFamily})` : "Auto" }, + ...Object.entries(ROUTER_FAMILIES).map(([value, label]) => ({ value, label })), + ]; + return ( +
+ {showPreferences && ( + <> +
+ +
+ {Object.entries(ROUTER_GOALS).map(([key, label]) => ( + + ))} +
+
+
+ + +
+ + )} + {waitingForTeam &&

Select a team to see available models.

} + {!waitingForTeam && loading && ( +

+ + Loading deployed models +

+ )} + {inventoryReady && error && ( +

+ Could not load deployed models.{" "} + +

+ )} + {inventoryReady && !error && ( + <> +
+
+ + {customized && hasSuggestions && ( + + )} +
+
+ {(["everyday", "hard"] as const).map((role) => { + const chosen = role === "everyday" ? everyday : hard; + const other = role === "everyday" ? hard : everyday; + const choices = models.filter((model) => model.name !== other?.name); + const items = choices.map((model) => ({ value: model.name, label: model.name })); + const label = role === "everyday" ? "Everyday" : "Hard tasks"; + return ( +
+
+ +

+ {role === "everyday" ? "Your cheaper model" : "Your stronger model"} +

+
+ + {chosen &&

{modelDescription(chosen)}

} +
+ ); + })} +
+
+ {(!everyday || !hard) && ( +

+ {models.length < 2 + ? "Two deployed models are needed. Add a model or use Advanced configuration." + : "Choose two models to get started."} +

+ )} + {notice && !customized && ( +

+ {notice} +

+ )} + + )} + +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx index ee9cbe3c480..52c4e9093a4 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx @@ -1,3 +1,4 @@ +import { autoRouterListKey } from "@/app/(dashboard)/hooks/models/useModels"; import { normalizeTierModels } from "./complexity_router_tiers"; import { openAutoRouterAdvanced, @@ -174,6 +175,12 @@ vi.mock("../common_components/team_dropdown", () => ({ ), })); +const renderAdvanced: typeof renderWithProviders = (...args) => { + const result = renderWithProviders(...args); + fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true })); + return result; +}; + const Harness = () => ; describe("AddAutoRouterTab", () => { @@ -192,12 +199,185 @@ describe("AddAutoRouterTab", () => { vi.mocked(modelCostMap).mockResolvedValue({}); }); + it("creates the goal-selected latest OpenAI pair using its Bedrock deployment aliases", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["small", "middle-old", "middle", "large"].map((model_group) => ({ model_group, mode: "chat" })), + ); + mockFetchAllModelDeployments.mockResolvedValue([ + { model_name: "small", litellm_params: { model: "bedrock/us.openai.gpt-6-luna" } }, + { model_name: "middle-old", litellm_params: { model: "openai/gpt-6-sol" } }, + { model_name: "middle", litellm_params: { model: "bedrock/us.openai.gpt-6.1-sol" } }, + { model_name: "large", litellm_params: { model: "bedrock/global.openai.gpt-6-astra" } }, + ]); + renderWithProviders(); + expect(await screen.findByRole("combobox", { name: "Everyday model" })).toHaveTextContent("small"); + expect(screen.queryByText("Define your tiers")).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Best quality", exact: true })); + expect(screen.getByRole("combobox", { name: "Everyday model" })).toHaveTextContent("middle"); + expect(screen.getAllByText("gpt-6.1-sol · Bedrock").some((element) => element.tagName === "P")).toBe(true); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0]).toMatchObject({ + auto_router_name: "auto-router", + complexity_router_config: { + classifier_type: "heuristic", + classification_mode: "user_turn", + tiers: { SIMPLE: ["middle"], MEDIUM: ["middle"], COMPLEX: ["large"], REASONING: ["large"] }, + }, + }); + }); + + it("creates a router from two custom models without a goal or family decision", async () => { + const customModels = ["fast-custom", "quality-custom"]; + mockFetchAvailableModels.mockResolvedValue(customModels.map((model_group) => ({ model_group, mode: "chat" }))); + mockFetchAllModelDeployments.mockResolvedValue( + customModels.map((model_name) => ({ + model_name, + litellm_params: { model: `openai/${model_name}` }, + })), + ); + renderWithProviders(); + expect(await screen.findByText("Choose two models to get started.")).toBeVisible(); + expect(screen.queryByRole("group", { name: "Routing goal" })).not.toBeInTheDocument(); + expect(screen.queryByRole("combobox", { name: "Model family" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + await chooseSelectOption( + userEvent.setup(), + screen.getByRole("combobox", { name: "Everyday model" }), + /fast-custom/, + ); + await chooseSelectOption( + userEvent.setup(), + screen.getByRole("combobox", { name: "Hard tasks model" }), + /quality-custom/, + ); + expect(screen.queryByRole("button", { name: "Use suggested models" })).not.toBeInTheDocument(); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0].complexity_router_config).toMatchObject({ + classifier_type: "heuristic", + classification_mode: "user_turn", + tiers: { + SIMPLE: ["fast-custom"], + MEDIUM: ["fast-custom"], + COMPLEX: ["quality-custom"], + REASONING: ["quality-custom"], + }, + }); + }); + + it("saves concrete solver names expanded by the backend from an openai wildcard route", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["openai/gpt-6-luna", "openai/gpt-6.1-sol", "openai/gpt-6-astra"].map((model_group) => ({ + model_group, + mode: "chat", + providers: ["openai"], + })), + ); + mockFetchAllModelDeployments.mockResolvedValue([{ model_name: "openai/*", litellm_params: { model: "openai/*" } }]); + renderWithProviders(); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls[0][0]).toMatchObject({ + complexity_router_config: { + classifier_type: "heuristic", + classification_mode: "user_turn", + tiers: { + SIMPLE: ["openai/gpt-6-luna"], + MEDIUM: ["openai/gpt-6-luna"], + COMPLEX: ["openai/gpt-6-astra"], + REASONING: ["openai/gpt-6-astra"], + }, + }, + }); + }); + + it.each(["groups", "deployments"])( + "keeps chosen models and saving available after a failed %s background refresh", + async (source) => { + mockFetchAvailableModels.mockResolvedValue( + ["claude-haiku-5-5", "claude-opus-5-5"].map((model_group) => ({ model_group, mode: "chat" })), + ); + renderWithProviders(); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + const fetch = source === "groups" ? mockFetchAvailableModels : mockFetchAllModelDeployments; + fetch.mockRejectedValue(new Error("Background refresh unavailable")); + const queryKey = + source === "groups" ? ["availableModels", "autoRouter", "token"] : autoRouterListKey("", "Admin"); + await act(async () => { + await testQueryClient.refetchQueries({ queryKey }); + }); + await waitFor(() => expect(testQueryClient.getQueryState(queryKey)?.status).toBe("error")); + expect(screen.getByRole("combobox", { name: "Everyday model" })).toHaveTextContent("claude-haiku-5-5"); + expect(screen.queryByText("Could not load deployed models.")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled(); + fireEvent.click(screen.getByRole("button", { name: "Add Auto Router" })); + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalledOnce()); + }, + ); + + it("keeps manual solver choices on goal changes and carries them into Advanced", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["claude-haiku-5-5", "custom-fast", "claude-opus-5-5"].map((model_group) => ({ + model_group, + mode: "chat", + })), + ); + renderWithProviders(); + const everyday = await screen.findByRole("combobox", { name: "Everyday model" }); + await userEvent.click(everyday); + await userEvent.click(await screen.findByRole("option", { name: /custom-fast/ })); + fireEvent.click(screen.getByRole("button", { name: "Cheapest", exact: true })); + expect(screen.getByRole("combobox", { name: "Everyday model" })).toHaveTextContent("custom-fast"); + expect(screen.getByRole("combobox", { name: "Hard tasks model" })).toHaveTextContent("claude-opus-5-5"); + fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true })); + expectTierModel("Simple", "custom-fast"); + expectTierModel("Reasoning", "claude-opus-5-5"); + }); + + it("keeps standard setup usable when the Heuristic v2 allowance is exhausted", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["claude-haiku-5-5", "claude-opus-5-5"].map((model_group) => ({ model_group, mode: "chat" })), + ); + vi.mocked(apiClient.post).mockResolvedValue({ + allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }], + error: null, + }); + renderWithProviders(); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + expect(screen.queryByText(/allowance/i)).not.toBeInTheDocument(); + }); + + it("blocks creation when deployment identity cannot be verified, then recovers on retry", async () => { + mockFetchAvailableModels.mockResolvedValue( + ["claude-haiku-5-5", "claude-opus-5-5"].map((model_group) => ({ model_group, mode: "chat" })), + ); + mockFetchAllModelDeployments.mockRejectedValueOnce(new Error("Unavailable")); + renderWithProviders(); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not load deployed models"); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Retry", exact: true })); + await waitFor(() => expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeEnabled()); + }); + + it("offers Advanced instead of inventing a second solver for a one-model deployment", async () => { + mockFetchAvailableModels.mockResolvedValue([{ model_group: "claude-opus-5-5", mode: "chat" }]); + renderWithProviders(); + expect(await screen.findByText(/Two deployed models are needed/)).toBeVisible(); + expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Advanced configuration", exact: true })); + expect(screen.getByText("Define your tiers")).toBeVisible(); + }); + it("shows Jev essentials before tiers and saves tuning from its own disclosure", async () => { mockFetchAvailableModels.mockResolvedValue([ ...ALL_FAMILY_MODELS, { model_group: "my-decision", mode: "evaluation" }, ]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await userEvent.click(screen.getByRole("radio", { name: "Decisions Model" })); expect(screen.queryByLabelText("Classifier Model")).not.toBeInTheDocument(); @@ -250,7 +430,7 @@ describe("AddAutoRouterTab", () => { { model_name: "ordinary-chat", litellm_params: { model: "mixed-provider/chat-model" } }, { model_name: "unavailable-decider", litellm_params: { model: underlyingModel } }, ]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await userEvent.click(screen.getByRole("radio", { name: "Decisions Model" })); await userEvent.click(screen.getByRole("combobox", { name: "Decision model" })); @@ -269,7 +449,7 @@ describe("AddAutoRouterTab", () => { it("uses the judge picker empty state when no decision deployments exist", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await userEvent.click(screen.getByRole("radio", { name: "Decisions Model" })); await userEvent.click(screen.getByRole("combobox", { name: "Decision model" })); @@ -280,7 +460,7 @@ describe("AddAutoRouterTab", () => { it("does not refill a tier cleared by the user when models refresh", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await userEvent.click(within(tierChips("Simple")).getByRole("button", { name: "" })); await act(async () => { @@ -294,7 +474,7 @@ describe("AddAutoRouterTab", () => { allowances: [{ key: "heuristic_v2", limit: 1, remaining, available: true }], error: null, }); - renderWithProviders(); + renderAdvanced(); await waitFor(() => expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Rule-based")); }); @@ -305,21 +485,21 @@ describe("AddAutoRouterTab", () => { ? "This change needs an available rule-based tuning allowance" : null, })); - renderWithProviders(); + renderAdvanced(); await waitFor(() => expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Rule-based")); expect(screen.queryByRole("alert")).not.toBeInTheDocument(); expect(screen.getByRole("button", { name: "Add Auto Router" })).toBeDisabled(); }); it("keeps the Rule-based default when the form reopens with a cached allowance", async () => { - const first = renderWithProviders(); + const first = renderAdvanced(); await waitFor(() => expect(apiClient.post).toHaveBeenCalled()); first.unmount(); vi.mocked(apiClient.post).mockResolvedValueOnce({ allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }], error: null, }); - renderWithProviders(); + renderAdvanced(); await waitFor(() => expect(screen.getByRole("button", { name: "Heuristic" })).toHaveTextContent("Rule-based")); }); @@ -330,7 +510,7 @@ describe("AddAutoRouterTab", () => { allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }], error: null, }); - renderWithProviders(); + renderAdvanced(); fireEvent.click(screen.getByRole("button", { name: "Heuristic" })); const option = screen.getByRole("menuitemradio", { name: /^Heuristic v2/ }); await waitFor(() => expect(option).toHaveTextContent("0 of 1 available")); @@ -383,7 +563,7 @@ describe("AddAutoRouterTab", () => { ...ALL_FAMILY_MODELS, { model_group: "my-decision", mode: "evaluation" }, ]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await user.click(screen.getByRole("radio", { name: "Decisions Model" })); await userEvent.click(screen.getByRole("combobox", { name: "Decision model" })); @@ -431,7 +611,7 @@ describe("AddAutoRouterTab", () => { ...ALL_FAMILY_MODELS, { model_group: "my-decision", mode: "evaluation" }, ]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await user.click(screen.getByRole("radio", { name: "Decisions Model" })); await userEvent.click(screen.getByRole("combobox", { name: "Decision model" })); @@ -468,7 +648,7 @@ describe("AddAutoRouterTab", () => { complete = resolve; }), ); - renderWithProviders(); + renderAdvanced(); await userEvent.click(screen.getByRole("radio", { name: "LLM" })); complete?.({ allowances: [{ key: "heuristic_v2", limit: 1, remaining: 0, available: true }], error: null }); await waitFor(() => expect(screen.getByRole("radio", { name: "LLM" })).toBeChecked()); @@ -484,7 +664,7 @@ describe("AddAutoRouterTab", () => { complete = resolve; }), ); - renderWithProviders(); + renderAdvanced(); await userEvent.click(screen.getByRole("radio", { name: family })); await selectAutoRouterOption("How often to classify", "Every new user message"); await act(async () => { @@ -508,8 +688,8 @@ describe("AddAutoRouterTab", () => { { model_group: "capable", mode: "chat" }, { model_group: "judge", mode: "chat" }, ]); - renderWithProviders(); - await user.type(screen.getByLabelText("Auto Router Name"), "forecast-router"); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "forecast-router" } }); await selectAutoRouterApproach(label); expect(screen.getByLabelText("Auto Router Name")).toHaveValue("forecast-router"); expect(screen.queryByTestId("template-selector")).not.toBeInTheDocument(); @@ -575,7 +755,7 @@ describe("AddAutoRouterTab", () => { it("restores the automatic/template/detail flow on the Complexity tab", async () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); await selectAutoRouterApproach("Capability"); expect(screen.queryByTestId("configure-automatically-button")).not.toBeInTheDocument(); @@ -603,9 +783,9 @@ describe("AddAutoRouterTab", () => { { model_group: "capable", mode: "chat" }, { model_group: "judge", mode: "chat" }, ]); - renderWithProviders(); + renderAdvanced(); await selectAutoRouterApproach(label); - await user.type(screen.getByLabelText("Auto Router Name"), "forecast-retry"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "forecast-retry" } }); const capability = label === "Capability"; const policyField = capability ? "Solve probability threshold" : "Efficient solver profile"; fireEvent.change(screen.getByLabelText(policyField), { @@ -627,7 +807,7 @@ describe("AddAutoRouterTab", () => { // Detailed Configuration starts collapsed so the modal opens onto just Name + Template; a caller // opts into the full tier/classifier form rather than always seeing it up front. it("shows models immediately and keeps advanced settings collapsed", async () => { - renderWithProviders(); + renderAdvanced(); expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false"); @@ -641,14 +821,14 @@ describe("AddAutoRouterTab", () => { { model_group: "unknown-model-a", mode: "chat" }, { model_group: "unknown-model-b", mode: "chat" }, ]); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => expect(optionByLabel("Anthropic Family")).toHaveTextContent("Missing:")); expect(screen.queryByTestId("configure-automatically-button")).not.toBeInTheDocument(); }); - it("mixes preferred tier models even when one complete preset is available", async () => { + it("prefers a same-family pair when a complete family is available", async () => { const anthropicPreset = getPresetByKey("anthropic_family")!; mockFetchAvailableModels.mockResolvedValue( [...getRequiredModelsInPreset(anthropicPreset), "gpt-5.6-luna"].map((model_group) => ({ @@ -657,18 +837,18 @@ describe("AddAutoRouterTab", () => { })), ); mockFetchAllModelDeployments.mockResolvedValue([]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); - expectTierModel("Simple", "gpt-5.6-luna"); - expectTierModel("Medium", "claude-sonnet-5"); + expectTierModel("Simple", ANTHROPIC_TIERS.SIMPLE[0]); + expectTierModel("Medium", ANTHROPIC_TIERS.SIMPLE[0]); expectTierModel("Complex", "claude-opus-5-5"); expectTierModel("Reasoning", "claude-opus-5-5"); expect(toast.success).not.toHaveBeenCalledWith(expect.stringContaining("Configured with")); }); - it("mixes available models from the preferred tier catalog when no complete template fits", async () => { + it("uses two models within the available family without a complete template", async () => { mockFetchAvailableModels.mockResolvedValue( ["gpt-5.6-luna", "claude-sonnet-5", "gpt-5.6-sol"].map((model_group) => ({ model_group, @@ -676,12 +856,12 @@ describe("AddAutoRouterTab", () => { })), ); mockFetchAllModelDeployments.mockResolvedValue([]); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); expectTierModel("Simple", "gpt-5.6-luna"); - expectTierModel("Medium", "claude-sonnet-5"); + expectTierModel("Medium", "gpt-5.6-luna"); expectTierModel("Complex", "gpt-5.6-sol"); expectTierModel("Reasoning", "gpt-5.6-sol"); }); @@ -689,20 +869,20 @@ describe("AddAutoRouterTab", () => { it("opens Detailed Configuration on the tiers automatic setup just filled in", async () => { const simpleModel = "gpt-5.6-luna"; mockFetchAvailableModels.mockResolvedValue([...ALL_FAMILY_MODELS, { model_group: simpleModel, mode: "chat" }]); - renderWithProviders(); + renderAdvanced(); expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false"); await waitForAutomaticTiers(); expect(screen.getByText("Define your tiers")).toBeInTheDocument(); - expectTierModel("Simple", simpleModel); + expectTierModel("Simple", ANTHROPIC_TIERS.SIMPLE[0]); }); // Nothing is filled in, so there is nothing to submit. The button reports that itself instead of // accepting a click and answering with a toast. it("offers no submit at all until every tier has a model", async () => { - renderWithProviders(); + renderAdvanced(); expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); }); @@ -710,7 +890,8 @@ describe("AddAutoRouterTab", () => { it("still flags the router name once the config no longer blocks the submit", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -720,15 +901,27 @@ describe("AddAutoRouterTab", () => { }); it("offers no team selector to a proxy admin, who may create an unscoped router", async () => { - renderWithProviders(); + renderAdvanced(); expect(screen.queryByTestId("team-dropdown")).not.toBeInTheDocument(); }); - it("requires a team admin to pick a team", async () => { + it("waits for a team before showing model inventory in simple setup", async () => { renderWithProviders( , ); + expect(screen.getByText("Select a team to see available models.")).toBeInTheDocument(); + expect(screen.queryByText(/Two deployed models are needed/)).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: /add auto router/i })).toBeDisabled(); + await userEvent.selectOptions(screen.getByTestId("team-dropdown"), "team-1"); + expect(await screen.findByText("Chosen models")).toBeInTheDocument(); + expect(screen.queryByText("Select a team to see available models.")).not.toBeInTheDocument(); + }); + + it("requires a team admin to pick a team", async () => { + renderAdvanced( + , + ); expect(screen.getByTestId("team-dropdown")).toBeInTheDocument(); expect(screen.getByText("Select Team")).toBeInTheDocument(); @@ -741,11 +934,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders( + renderAdvanced( , ); - await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "team-scoped-router" } }); await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1"); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -763,7 +956,7 @@ describe("AddAutoRouterTab", () => { ); vi.mocked(handleAddAutoRouterSubmit).mockImplementationOnce(actualSubmit.handleAddAutoRouterSubmit); const user = userEvent.setup(); - renderWithProviders( + renderAdvanced( { error: "session_affinity cannot be combined with tier_definitions", }); - renderWithProviders(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "rejected-router"); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "rejected-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -838,7 +1031,7 @@ describe("AddAutoRouterTab", () => { }), ); - const { container } = renderWithProviders(); + const { container } = renderAdvanced(); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "double-submit-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); @@ -858,8 +1051,8 @@ describe("AddAutoRouterTab", () => { vi.mocked(getMissingTiersError).mockReturnValue(null); validateAutoRouterConfig.mockResolvedValueOnce({ valid: true }); - renderWithProviders(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "accepted-router"); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "accepted-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -873,9 +1066,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "keyword-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -890,9 +1083,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "keyword-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -908,9 +1101,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "orphan-rule-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "orphan-rule-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -927,9 +1120,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "keyword-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -944,9 +1137,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "keyword-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -965,11 +1158,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders( + renderAdvanced( , ); - await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "team-scoped-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -983,11 +1176,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders( + renderAdvanced( , ); - await user.type(screen.getByPlaceholderText(/smart_router/i), "team-scoped-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "team-scoped-router" } }); await user.selectOptions(screen.getByTestId("team-dropdown"), "team-1"); await user.click(screen.getByTestId("team-dropdown-clear")); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); @@ -1001,9 +1194,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "affinity-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); expect(await screen.findByRole("combobox", { name: "How often to classify" })).not.toHaveTextContent( @@ -1022,7 +1215,8 @@ describe("AddAutoRouterTab", () => { it("blocks invalid success thresholds and creates a heuristic v2 router with explicit zero", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "" } }); fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "threshold-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); @@ -1059,7 +1253,7 @@ describe("AddAutoRouterTab", () => { "./handle_add_auto_router_submit", ); vi.mocked(handleAddAutoRouterSubmit).mockImplementationOnce(actualSubmit.handleAddAutoRouterSubmit); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "chained-router" } }); await userEvent.click(screen.getByRole("radio", { name: "LLM" })); @@ -1087,7 +1281,7 @@ describe("AddAutoRouterTab", () => { it("preserves classifier tuning when choosing models automatically", async () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "reset-threshold-router" } }); openAutoRouterAdvanced("Classification Method"); @@ -1112,7 +1306,8 @@ describe("AddAutoRouterTab", () => { it("clears an invalid inactive threshold before creating the router", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "" } }); fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "clear-threshold-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); @@ -1133,13 +1328,15 @@ describe("AddAutoRouterTab", () => { it.each([false, true])("creates a router with cache routing only after an explicit opt-in: %s", async (enabled) => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForAutomaticTiers(); fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "cache-aware-router" } }); openAutoRouterAdvanced("Cache-aware routing"); const toggle = screen.getByRole("switch", { name: "Cache-aware routing" }); expect(toggle).not.toBeChecked(); if (enabled) { + expect(toggle).toHaveAttribute("aria-disabled", "true"); + await selectAutoRouterOption("How often to classify", "Every request"); await user.click(toggle); fireEvent.change(screen.getByLabelText("Expected output tokens"), { target: { value: "512" } }); fireEvent.change(screen.getByLabelText("Prediction timeout (ms)"), { target: { value: "750" } }); @@ -1158,9 +1355,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-window-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "ctx-window-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Context Window Escalation"); const toggle = await screen.findByRole("switch", { name: "Escalate oversized prompts to a tier that fits" }); @@ -1181,9 +1378,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-buffer-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "ctx-buffer-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Context Window Escalation"); await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" })); @@ -1204,9 +1401,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "ctx-clear-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "ctx-clear-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Context Window Escalation"); await user.click(screen.getByRole("switch", { name: "Escalate oversized prompts to a tier that fits" })); @@ -1230,9 +1427,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "no-compression-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "no-compression-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1246,9 +1443,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "no-compression-explicit-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { + target: { value: "no-compression-explicit-router" }, + }); expandDetailedConfiguration(); openAutoRouterAdvanced("Compression"); await chooseSelectOption( @@ -1270,9 +1469,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "different-compression-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { + target: { value: "different-compression-router" }, + }); expandDetailedConfiguration(); openAutoRouterAdvanced("Compression"); await chooseSelectOption( @@ -1299,9 +1500,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "override-floor-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "override-floor-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); await selectAutoRouterOption("Heuristic", "Rule-based"); @@ -1321,9 +1522,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "affinity-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); await selectAutoRouterOption("How often to classify", "Once per session"); @@ -1346,9 +1547,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "user-turn-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "user-turn-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); await selectAutoRouterOption("How often to classify", "Every new user message"); @@ -1362,16 +1563,19 @@ describe("AddAutoRouterTab", () => { }); }); - it("writes every_request into the create payload when the default frequency stays selected", async () => { + it("allows every request in Advanced while defaulting to every new user message", async () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "default-timing-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "default-timing-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Classification Method"); - expect(await screen.findByRole("combobox", { name: "How often to classify" })).toHaveTextContent("Every request"); + expect(await screen.findByRole("combobox", { name: "How often to classify" })).toHaveTextContent( + "Every new user message", + ); + await selectAutoRouterOption("How often to classify", "Every request"); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1386,9 +1590,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "affinity-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Affinity"); expect(await screen.findByRole("switch", { name: "Pin one model deployment per tier" })).toBeChecked(); @@ -1406,9 +1610,9 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "affinity-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "affinity-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Affinity"); await user.click(await screen.findByRole("switch", { name: "Pin one model deployment per tier" })); @@ -1426,7 +1630,7 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "modality-router" } }); @@ -1444,7 +1648,7 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); vi.mocked(getMissingTiersError).mockReturnValue(null); - renderWithProviders(); + renderAdvanced(); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "modality-router" } }); expandDetailedConfiguration(); @@ -1465,7 +1669,7 @@ describe("AddAutoRouterTab", () => { // Custom is the escape hatch, not the headline choice, so it's listed after every bundled preset // rather than first. it("lists Start from scratch after the bundled presets", async () => { - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); const labels = visibleOptions().map((option) => option.querySelector(".font-medium")?.textContent); @@ -1487,7 +1691,7 @@ describe("AddAutoRouterTab", () => { ); vi.mocked(getMissingTiersError).mockImplementation(actual.getMissingTiersError); - renderWithProviders(); + renderAdvanced(); expect(screen.getByTestId("auto-router-test-routing-btn")).toBeDisabled(); }); @@ -1504,8 +1708,8 @@ describe("AddAutoRouterTab", () => { }, }); - renderWithProviders(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "keyword-router"); + renderAdvanced(); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "keyword-router" } }); expandDetailedConfiguration(); openAutoRouterAdvanced("Keyword/Semantic Matching"); await user.click(screen.getByRole("button", { name: /add keyword rule/i })); @@ -1540,7 +1744,7 @@ describe("AddAutoRouterTab", () => { }, }); - renderWithProviders(); + renderAdvanced(); await user.click(screen.getByTestId("auto-router-test-routing-btn")); await user.type(await screen.findByTestId("auto-router-routing-test-prompt"), "reconcile this invoice"); await user.click(screen.getByTestId("auto-router-routing-test-send")); @@ -1564,7 +1768,7 @@ describe("AddAutoRouterTab", () => { }), ); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); const anthropicOption = optionByLabel("Anthropic Family")!; @@ -1582,7 +1786,7 @@ describe("AddAutoRouterTab", () => { it("disables every preset and offers a retry when the model list fails to load", async () => { mockFetchAvailableModels.mockRejectedValue(new Error("network error")); - renderWithProviders(); + renderAdvanced(); expect(await screen.findByText("Could not load available models.")).toBeInTheDocument(); openTemplateDropdown(); @@ -1595,7 +1799,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); mockFetchAllModelDeployments.mockRejectedValue(new Error("network error")); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await waitForPresetEnabled("OpenAI Family"); @@ -1606,7 +1810,7 @@ describe("AddAutoRouterTab", () => { ALL_FAMILY_MODELS.filter((m) => m.model_group !== ANTHROPIC_ONLY_MODEL), ); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { @@ -1618,7 +1822,7 @@ describe("AddAutoRouterTab", () => { it("enables a preset once every model it needs is available", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await waitForPresetEnabled("OpenAI Family"); @@ -1626,7 +1830,7 @@ describe("AddAutoRouterTab", () => { it("shows the template's model choices while advanced settings stay collapsed", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); expect(screen.getByRole("button", { name: "Advanced settings" })).toHaveAttribute("aria-expanded", "false"); @@ -1635,7 +1839,7 @@ describe("AddAutoRouterTab", () => { }); it("expands detailed configuration when Start from scratch is chosen", async () => { - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await selectTemplate("Start from scratch"); @@ -1647,7 +1851,7 @@ describe("AddAutoRouterTab", () => { it("lets a caller manually re-expand a detailed configuration a preset just collapsed", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); expect(screen.queryByText("Keyword/Semantic Matching")).not.toBeInTheDocument(); @@ -1666,11 +1870,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); - await user.type(screen.getByPlaceholderText(/smart_router/i), "anthropic-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "anthropic-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1689,7 +1893,7 @@ describe("AddAutoRouterTab", () => { it("blocks a form submit when a referenced model disappears after the tiers are filled in", async () => { mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - const { container } = renderWithProviders(); + const { container } = renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "stale-model-router" } }); @@ -1713,11 +1917,11 @@ describe("AddAutoRouterTab", () => { const user = userEvent.setup(); mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); - await user.type(screen.getByPlaceholderText(/smart_router/i), "anthropic-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "anthropic-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1751,10 +1955,10 @@ describe("AddAutoRouterTab", () => { it("submits the pinned model in place of the one the tiers derive", async () => { const user = userEvent.setup(); - renderWithProviders(); + renderAdvanced(); await applyPresetAndPin(user); - await user.type(screen.getByPlaceholderText(/smart_router/i), "pinned-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "pinned-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1771,7 +1975,7 @@ describe("AddAutoRouterTab", () => { it("blocks a submit whose pinned model is no longer available", async () => { const user = userEvent.setup(); - const { container } = renderWithProviders(); + const { container } = renderAdvanced(); await applyPresetAndPin(user); fireEvent.change(screen.getByPlaceholderText(/smart_router/i), { target: { value: "stale-pin-router" } }); @@ -1796,11 +2000,11 @@ describe("AddAutoRouterTab", () => { it("omits plan_mode_min_tier from the payload when never touched", async () => { const user = userEvent.setup(); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); - await user.type(screen.getByPlaceholderText(/smart_router/i), "no-plan-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "no-plan-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1812,7 +2016,7 @@ describe("AddAutoRouterTab", () => { it("carries the enabled override through to the create payload", async () => { const user = userEvent.setup(); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Anthropic Family"); await selectTemplate("Anthropic Family"); @@ -1820,7 +2024,7 @@ describe("AddAutoRouterTab", () => { openAutoRouterAdvanced("Plan-Mode Override"); await user.click(await screen.findByRole("switch", { name: "Route plan-mode requests to a minimum tier" })); - await user.type(screen.getByPlaceholderText(/smart_router/i), "plan-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "plan-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1847,7 +2051,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(groupsFor(ALL_RENAMED_DEPLOYMENTS)); mockFetchAllModelDeployments.mockResolvedValue(ALL_RENAMED_DEPLOYMENTS); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { @@ -1874,7 +2078,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(groupsFor(deployments)); mockFetchAllModelDeployments.mockResolvedValue(deployments); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); @@ -1886,7 +2090,7 @@ describe("AddAutoRouterTab", () => { expect(screen.getByText("Keyword/Semantic Matching")).toBeInTheDocument(); - await user.type(screen.getByPlaceholderText(/smart_router/i), "renamed-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "renamed-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -1908,7 +2112,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(groupsFor(anthropicOnly)); mockFetchAllModelDeployments.mockResolvedValue(anthropicOnly); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { @@ -1933,7 +2137,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(groupsFor(wildcard)); mockFetchAllModelDeployments.mockResolvedValue(wildcard); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { @@ -1960,7 +2164,7 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(EXPANDED_HUB_GROUPS); mockFetchAllModelDeployments.mockResolvedValue(WILDCARD_DEPLOYMENTS); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { @@ -1974,14 +2178,14 @@ describe("AddAutoRouterTab", () => { mockFetchAvailableModels.mockResolvedValue(EXPANDED_HUB_GROUPS); mockFetchAllModelDeployments.mockResolvedValue(WILDCARD_DEPLOYMENTS); - renderWithProviders(); + renderAdvanced(); openTemplateDropdown(); await waitFor(() => { expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); }); await selectTemplate("Anthropic Family"); - await user.type(screen.getByPlaceholderText(/smart_router/i), "wildcard-router"); + fireEvent.change(screen.getByLabelText("Auto Router Name"), { target: { value: "wildcard-router" } }); await waitFor(() => expect(screen.getByRole("button", { name: /add auto router/i })).toBeEnabled()); await user.click(screen.getByRole("button", { name: /add auto router/i })); @@ -2079,7 +2283,7 @@ describe("preset catalog fetch states", () => { }, ], }); - renderWithProviders(); + renderAdvanced(); await waitForPresetEnabled("Bounded JEV"); await selectTemplate("Bounded JEV"); await userEvent.click(screen.getByRole("combobox", { name: "Decision model" })); @@ -2100,7 +2304,7 @@ describe("preset catalog fetch states", () => { ...LOADED_PRESETS_QUERY, isError: true, } as never); - renderWithProviders(); + renderAdvanced(); expect(screen.queryByText(/Could not load templates/)).not.toBeInTheDocument(); @@ -2114,7 +2318,7 @@ describe("preset catalog fetch states", () => { data: undefined, isPending: true, } as never); - renderWithProviders(); + renderAdvanced(); expect(screen.getByText("Loading templates...")).toBeInTheDocument(); }); @@ -2127,7 +2331,7 @@ describe("preset catalog fetch states", () => { isError: true, refetch, } as never); - renderWithProviders(); + renderAdvanced(); expect(await screen.findByText(/Could not load templates/)).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 82c30395755..8c3f402df1f 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -13,7 +13,7 @@ import { FormField } from "@/components/shared/form/FormField"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; -import { Select, SelectContent, SelectItem, SelectSeparator, SelectTrigger, SelectValue } from "@/components/ui/select"; +import AutoRouterPresetSelect, { type PresetAvailability } from "./AutoRouterPresetSelect"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { useZodForm } from "@/lib/forms/useZodForm"; @@ -79,7 +79,16 @@ import { } from "@/lib/autorouter_presets"; import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; -import { buildAutomaticRouterConfig, buildPreferredTierModels, prefillEmptyTiers } from "./auto_setup"; +import AutoRouterSimpleSetup from "./AutoRouterSimpleSetup"; +import { + deployedRouterModels, + nextRouterName, + simpleSetupError, + recommendSolverPair, + solverPairConfig, + type RouterGoal, + type FamilyPreference, +} from "./auto_setup"; interface AddAutoRouterTabProps { handleOk: () => void; @@ -95,31 +104,6 @@ interface AddAutoRouterTabProps { teams?: Team[] | null; } -type PresetAvailability = - | { kind: "available"; viaDeployments: boolean } - | { kind: "loading" } - | { kind: "unverifiable" } - | { kind: "missing_models"; models: readonly string[] }; - -// Every non-"available" state disables the option. Selection derives from this same function -// (see presetAvailability below), so an option a caller can click is always one that can be applied. -const presetDisabledHint = (availability: PresetAvailability): string | null => { - switch (availability.kind) { - case "available": - return null; - case "loading": - return "Checking model availability..."; - case "unverifiable": - return "Cannot verify these models are available"; - case "missing_models": - return `Missing: ${availability.models.join(", ")}`; - } -}; - -// "loading"/"unverifiable" are transient system states, not a gap specific to this preset; only a -// caller-specific missing-model reason gets the alarming red treatment. -const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models"; - const NO_PRESETS: AutoRouterPreset[] = []; // Why the submit is unavailable, or null when it is available. The button reads this to disable @@ -168,7 +152,7 @@ const autoRouterSchema = (requiresTeamScope: boolean) => type AddAutoRouterFormValues = z.infer>; const EMPTY_FORM_VALUES: AddAutoRouterFormValues = { - auto_router_name: "", + auto_router_name: "auto-router", team_id: null, model_access_group: undefined, }; @@ -210,11 +194,18 @@ const AddAutoRouterTab: React.FC = ({ }); const [modelAccessGroups, setModelAccessGroups] = useState([]); - const [complexityRouterConfig, setComplexityRouterConfig] = useState({ + const [advancedConfig, setComplexityRouterConfig] = useState({ tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, classifier_type: "heuristic", + classification_mode: "user_turn", }); + const [advanced, setAdvanced] = useState(false); + const [prefillAttempted, setPrefillAttempted] = useState(false); + const [goal, setGoal] = useState("balanced"); + const [family, setFamily] = useState("auto"); + const [solverOverrides, setSolverOverrides] = useState<{ everyday?: string; hard?: string }>({}); + const [customTechnicalKeywords, setCustomTechnicalKeywords] = useState([]); const [keywordTierRules, setKeywordTierRules] = useState([]); const [semanticMatchingEnabled, setSemanticMatchingEnabled] = useState(false); @@ -250,17 +241,30 @@ const AddAutoRouterTab: React.FC = ({ isError: modelsError, refetch: refetchModels, } = useQuery({ - queryKey: ["availableModels", "autoRouter", accessToken, ...(isMemberManaged ? [watchedTeamId] : [])], + queryKey: ["availableModels", "autoRouter", accessToken, ...(requiresTeamScope ? [watchedTeamId] : [])], queryFn: () => - isMemberManaged ? fetchAutoRouterModels(accessToken, watchedTeamId) : fetchAvailableModels(accessToken), - enabled: Boolean(accessToken && (!isMemberManaged || watchedTeamId)), + requiresTeamScope ? fetchAutoRouterModels(accessToken, watchedTeamId) : fetchAvailableModels(accessToken), + enabled: Boolean(accessToken && (!requiresTeamScope || watchedTeamId)), }); - const { data: deployments, isLoading: deploymentsLoading } = useQuery({ + const { + data: deployments, + isLoading: deploymentsLoading, + isError: deploymentsError, + refetch: refetchDeployments, + } = useQuery({ queryKey: autoRouterListKey(userId ?? "", userRole), queryFn: () => fetchAllModelDeployments(accessToken, userId ?? "", userRole), enabled: Boolean(accessToken), }); const modelsLoading = groupsLoading || deploymentsLoading; + useEffect(() => { + if (!deployments || form.getFieldState("auto_router_name").isDirty) return; + form.setValue( + "auto_router_name", + nextRouterName(deployments.flatMap((row) => (row.model_name ? [row.model_name] : []))), + ); + }, [deployments, form]); + const modelInfo = React.useMemo(() => data ?? [], [data]); const { data: presetsData, @@ -269,12 +273,12 @@ const AddAutoRouterTab: React.FC = ({ refetch: refetchPresets, } = useAutoRouterPresets(); const presets = presetsData ?? NO_PRESETS; - const automaticSetupLoading = modelsLoading || presetsPending; const presetsUnavailable = presetsError && presetsData === undefined; // react-query keeps the last successful list around when a later refetch fails, so isError alone // can't tell "never loaded" apart from "loaded, then a background refetch errored" - only the // former leaves us with nothing trustworthy to verify a preset's models against. const modelsUnverifiable = modelsError && data === undefined; + const inventoryUnavailable = modelsUnverifiable || (deploymentsError && deployments === undefined); const isAdmin = all_admin_roles.includes(userRole); @@ -294,14 +298,33 @@ const AddAutoRouterTab: React.FC = ({ ), [modelInfo], ); - const preferredTierModels = React.useMemo( - () => buildPreferredTierModels(presets, availability), - [presets, availability], + const solverModels = React.useMemo( + () => deployedRouterModels(modelInfo, deployments ?? []), + [modelInfo, deployments], ); - const automaticRouterConfig = React.useMemo( - () => buildAutomaticRouterConfig(modelInfo, deployments ?? [], preferredTierModels), - [modelInfo, deployments, preferredTierModels], + const recommended = React.useMemo( + () => recommendSolverPair(solverModels, goal, family), + [solverModels, goal, family], ); + const everyday = solverOverrides.everyday + ? solverModels.find((model) => model.name === solverOverrides.everyday) + : recommended?.everyday; + const hard = solverOverrides.hard + ? solverModels.find((model) => model.name === solverOverrides.hard) + : recommended?.hard; + const simpleConfig = everyday && hard ? solverPairConfig(everyday.name, hard.name) : null; + const complexityRouterConfig = advanced + ? advancedConfig + : simpleConfig ?? { + classifier_type: "heuristic" as const, + classification_mode: "user_turn" as const, + tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, + }; + const openAdvanced = () => { + setComplexityRouterConfig(complexityRouterConfig); + setPrefillAttempted(simpleConfig !== null); + setAdvanced(true); + }; // A preset's models can only be trusted against a successfully loaded list. Selection and the // greyed-out state derive from this one function, so a preset that cannot be selected can never @@ -322,22 +345,6 @@ const AddAutoRouterTab: React.FC = ({ [modelsLoading, modelsUnverifiable, availability, groupsOnlyAvailability], ); - const sortedPresetOptions = React.useMemo( - () => - presets - .map((preset) => ({ preset, availability: presetAvailability(preset) })) - .sort((a, b) => Number(b.availability.kind === "available") - Number(a.availability.kind === "available")), - [presets, presetAvailability], - ); - - const templateItems = React.useMemo( - () => [ - ...sortedPresetOptions.map(({ preset }) => ({ value: preset.key, label: preset.label })), - { value: "custom", label: "Start from scratch" }, - ], - [sortedPresetOptions], - ); - const applyPrefill = (prefill: PresetPrefill) => { setEditingTiers(false); setComplexityRouterConfig(prefill.complexityRouterConfig); @@ -349,16 +356,20 @@ const AddAutoRouterTab: React.FC = ({ setEscalationKeywords(prefill.escalationKeywords); }; - const [prefillAttempted, setPrefillAttempted] = useState(false); - const automaticPrefill = automaticSetupLoading || modelsUnverifiable ? null : automaticRouterConfig; - if (automaticPrefill && !prefillAttempted) { + if (advanced && !prefillAttempted && !modelsLoading && simpleConfig) { setPrefillAttempted(true); - setComplexityRouterConfig((current) => prefillEmptyTiers(current, automaticPrefill)); + setComplexityRouterConfig((current) => + !current.custom_tier_set && + !isForecastClassifier(current.classifier_type) && + activeTierRows(current).every((row) => row.models.length === 0) + ? { ...current, tiers: simpleConfig.tiers } + : current, + ); } const handlePresetChange = (presetKey: string | undefined) => { + setPrefillAttempted(true); if (!presetKey || presetKey === "custom") { - setPrefillAttempted(true); setSelectedPreset(presetKey); applyPrefill(buildEmptyPrefill()); @@ -372,7 +383,6 @@ const AddAutoRouterTab: React.FC = ({ const presetState = presetAvailability(preset); if (presetState.kind !== "available") return; - setPrefillAttempted(true); setSelectedPreset(presetKey); applyPrefill(buildPresetPrefill(preset.complexity_router_config, availability)); }; @@ -413,7 +423,11 @@ const AddAutoRouterTab: React.FC = ({ }, !requiresTeamScope || Boolean(watchedTeamId), ); - const saveBlockedReason = submitBlockedReason ?? routerAvailability.saveBlockedReason; + const simpleBlockedReason = advanced ? null : simpleSetupError(modelsLoading, inventoryUnavailable, everyday, hard); + const saveBlockedReason = + !advanced && requiresTeamScope && !watchedTeamId + ? "Select a team to see available models." + : simpleBlockedReason ?? submitBlockedReason ?? routerAvailability.saveBlockedReason; const handleClassifierChange = (config: ComplexityRouterConfigValue) => { setSelectedPreset(undefined); setComplexityRouterConfig(config); @@ -540,71 +554,20 @@ const AddAutoRouterTab: React.FC = ({ setIsTestModalVisible(true); }; - const tierSetup = ( -
-
- -

Replaces classifier and model choices

- {presetsPending &&
Loading templates...
} - {presetsUnavailable && ( -
- Could not load templates, so only Start from scratch is shown.{" "} - -
- )} -
-
- ); - const configurationForm = ( void refetchPresets()} + /> + } editingTiers={editingTiers} onEditingTiersChange={setEditingTiers} modelInfo={modelInfo} @@ -646,8 +609,8 @@ const AddAutoRouterTab: React.FC = ({ )} -
- {isAdmin && ( +
+ {isAdmin && advanced && (
= ({ )} - - } - judgeModel={ - model.mode !== "embedding") - .map((model) => ({ value: model.model_group, label: model.model_group }))} - showValidationErrors={showValidationErrors} - /> - } - > - - {modelsUnverifiable && ( -

- Could not load available models.{" "} - -

- )} - {configurationForm} - -
- - - Need Help? - - } - /> - Get help on our github - -
- - - - +

+ )} + {configurationForm} + + + )} + {!advanced && routerAvailability.isError && ( +

+ Could not check availability.{" "} + +

+ )} + {!advanced && routerAvailability.data?.error && ( +

+ {routerAvailability.data.error} +

+ )} +
+ + - {isTestingConnection && } - Test Connection - - - - -
-
- - + Need Help? + + } + /> + Get help on our github + +
+ + + + + + + +
+
diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts index 7b12a0d54fd..4097afc17fe 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts @@ -1,274 +1,325 @@ -import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; +import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; -import { buildModelAvailability, deploymentRefsFromModelInfo } from "@/lib/autorouter_presets"; import { - buildAutomaticRouterConfig, - buildPreferredTierModels, - prefillEmptyTiers, - type PreferredTierModels, + deployedRouterModels, + recommendSolverPair, + solverPairConfig, + nextRouterName, + simpleSetupError, } from "./auto_setup"; -const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" })); -const reasoningModel = (model_group: string, supported_reasoning_efforts: string[]) => ({ - model_group, - mode: "chat", - supports_reasoning: true, - supported_reasoning_efforts, -}); -const deployment = (model_name: string, model = model_name): AutoRouterDeployment => ({ - model_name, +const deployment = (name: string, model: string, extra: Partial = {}): AutoRouterDeployment => ({ + model_name: name, litellm_params: { model }, + ...extra, }); -const tierModels = (config: ReturnType) => - config && Object.values(config.tiers).map((tier) => (typeof tier === "string" ? tier : tier[0])); +const resolve = (...rows: AutoRouterDeployment[]) => + deployedRouterModels( + rows.map((row) => ({ model_group: row.model_name!, mode: "chat" })), + rows, + ); +const names = (pair: ReturnType) => pair && [pair.everyday.name, pair.hard.name]; +const ladder = () => + resolve( + deployment("small", "anthropic/claude-haiku-5-5"), + deployment("medium", "anthropic/claude-sonnet-5-5"), + deployment("large", "anthropic/claude-opus-5-5"), + ); -describe("buildPreferredTierModels", () => { - it("recognizes curated models that are not in a preset", () => { - const available = ["gpt-5.6-luna", "claude-sonnet-5", "grok-4.6", "deepseek-v4-pro"]; - const availability = buildModelAvailability(available, []); - const preferred = buildPreferredTierModels([], availability); - const expected: PreferredTierModels = { - SIMPLE: ["gpt-5.6-luna"], - MEDIUM: ["claude-sonnet-5"], - COMPLEX: ["deepseek-v4-pro", "grok-4.6"], - REASONING: ["deepseek-v4-pro", "grok-4.6"], - }; +const catalog: Record = JSON.parse( + readFileSync(new URL("../../../../../model_prices_and_context_window.json", import.meta.url), "utf8"), +); +const catalogOpusModels = Object.entries(catalog).filter( + ([id, metadata]) => metadata.mode === "chat" && /claude-opus-\d/.test(id) && !/-(?:fast|think)$/.test(id), +); - expect(preferred).toEqual(expected); - }); - - it("prefers the current model ladder over older preset entries", () => { - const availability = buildModelAvailability(["gpt-5.6-luna", "gpt-4o-mini"], []); - const preferred = buildPreferredTierModels( - [ - { - key: "old", - label: "Old", - description: "Old model", - complexity_router_config: { - tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] }, - classifier_type: "heuristic_v2", - }, - }, - ], - availability, - ); - - expect(preferred.SIMPLE).toEqual(["gpt-5.6-luna", "gpt-4o-mini"]); - }); -}); - -describe("buildAutomaticRouterConfig", () => { - it("selects native Terra and Sol groups with their reasoning settings, retaining cloud fallback", () => { - const modelNames = ["gpt-5.6-terra", "gpt-5.6-sol"]; - const deployments = modelNames.flatMap((model) => [ - deployment(model, `azure/${model}`), - deployment(`z-native-${model}`, `openai/${model}`), - ]); - const available = deployments.map(({ model_name }) => reasoningModel(model_name!, ["none", "high"])); - const availability = buildModelAvailability( - available.map(({ model_group }) => model_group), - deploymentRefsFromModelInfo(deployments), - ); - const preferred = buildPreferredTierModels([], availability); - const config = buildAutomaticRouterConfig(available, deployments, preferred); - - expect(tierModels(config)).toEqual([ - "z-native-gpt-5.6-terra", - "z-native-gpt-5.6-terra", - "z-native-gpt-5.6-sol", - "z-native-gpt-5.6-sol", - ]); - expect(config?.tier_model_params).toEqual({ - REASONING: { "z-native-gpt-5.6-sol": { reasoning_effort: "high" } }, - }); - - const cloudOnly = models(...modelNames); - const cloudAvailability = buildModelAvailability(modelNames, deploymentRefsFromModelInfo(deployments)); - const cloudPreferred = buildPreferredTierModels([], cloudAvailability); - - expect(tierModels(buildAutomaticRouterConfig(cloudOnly, deployments, cloudPreferred))).toEqual([ - "gpt-5.6-terra", - "gpt-5.6-terra", - "gpt-5.6-sol", - "gpt-5.6-sol", - ]); - }); - - it("selects one preferred model for each tier", () => { - const preferred: PreferredTierModels = { - SIMPLE: ["simple"], - MEDIUM: ["medium"], - COMPLEX: ["complex"], - REASONING: ["reasoning"], - }; - - expect( - tierModels(buildAutomaticRouterConfig(models("simple", "medium", "complex", "reasoning"), [], preferred)), - ).toEqual(["simple", "medium", "complex", "reasoning"]); - }); - - it.each([ - { - provider: "OpenAI", - available: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra"], - expected: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra", "gpt-6-astra"], - supportedEfforts: ["low", "medium", "high", "xhigh", "max"], - effort: "max", - }, - { - provider: "Anthropic", - available: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"], - expected: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5", "claude-opus-5"], - supportedEfforts: ["low", "medium", "high", "max"], - effort: "max", - }, - { - provider: "Google", - available: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview"], - expected: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3.1-pro-preview"], - supportedEfforts: ["low", "medium", "high"], - effort: "high", - }, - { - provider: "DeepSeek", - available: ["deepseek-v4-flash", "deepseek-v4-pro"], - expected: ["deepseek-v4-flash", "deepseek-v4-flash", "deepseek-v4-pro", "deepseek-v4-pro"], - supportedEfforts: ["none", "high"], - effort: "high", - }, - { - provider: "xAI", - available: ["grok-4.6"], - expected: ["grok-4.6", "grok-4.6", "grok-4.6", "grok-4.6"], - supportedEfforts: ["low", "medium", "high", "xhigh"], - effort: "xhigh", - }, - ])( - "uses the current $provider ladder and strongest advertised reasoning effort", - ({ available, expected, supportedEfforts, effort }) => { - const availability = buildModelAvailability(available, []); - const preferred = buildPreferredTierModels([], availability); - const modelInfo = models(...available).map((model) => - model.model_group === expected[3] ? reasoningModel(model.model_group, supportedEfforts) : model, - ); - - const config = buildAutomaticRouterConfig(modelInfo, [], preferred); - - expect(tierModels(config)).toEqual(expected); - expect(config?.tier_model_params).toEqual({ - REASONING: { [expected[3]]: { reasoning_effort: effort } }, - }); +describe("deployedRouterModels", () => { + it.each(catalogOpusModels)( + "recognizes the catalog's standard Opus deployment %s behind a custom alias", + (id, metadata) => { + const row = { + model_name: "custom-deployment", + litellm_params: { model: id, custom_llm_provider: metadata.litellm_provider }, + }; + const models = deployedRouterModels([{ model_group: "custom-deployment", mode: "chat" }], [row]); + expect(models).toEqual([expect.objectContaining({ name: "custom-deployment", family: "claude", line: "opus" })]); }, ); - it("never exceeds the selected model group's advertised reasoning efforts", () => { - const available = ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol"]; - const availability = buildModelAvailability(available, []); - const preferred = buildPreferredTierModels([], availability); - const modelInfo = [ - ...models("gpt-5.6-luna", "gpt-5.6-terra"), - reasoningModel("gpt-5.6-sol", ["none", "low", "medium", "high", "xhigh"]), - ]; - - const config = buildAutomaticRouterConfig(modelInfo, [], preferred); - - expect(config?.tier_model_params).toEqual({ - REASONING: { "gpt-5.6-sol": { reasoning_effort: "xhigh" } }, - }); + it("resolves underlying identities independently of deployment aliases and serving providers", () => { + const models = resolve( + deployment("everyday", "bedrock/us.openai.gpt-6.1-sol"), + deployment("hard", "bedrock_converse/global.openai.gpt-6-astra"), + ); + const expectedEveryday = { name: "everyday", family: "openai", line: "sol", version: [6, 1], provider: "Bedrock" }; + const expectedHard = { name: "hard", family: "openai", line: "astra", version: [6], provider: "Bedrock" }; + expect(models).toEqual( + expect.arrayContaining([expect.objectContaining(expectedEveryday), expect.objectContaining(expectedHard)]), + ); + expect(names(recommendSolverPair(models, "quality", "openai"))).toEqual(["everyday", "hard"]); }); - it("leaves reasoning effort unset when the proxy does not report supported values", () => { - const available = ["grok-4.6"]; - const availability = buildModelAvailability(available, []); - const preferred = buildPreferredTierModels([], availability); - - const config = buildAutomaticRouterConfig(models(...available), [], preferred); - - expect(config?.tier_model_params).toBeUndefined(); - }); - - it("reuses the closest available tier when a tier has no match", () => { - const preferred: PreferredTierModels = { - SIMPLE: ["simple"], - MEDIUM: [], - COMPLEX: ["complex"], - REASONING: [], - }; - - expect(tierModels(buildAutomaticRouterConfig(models("simple", "complex"), [], preferred))).toEqual([ - "simple", - "simple", - "complex", - "complex", + it("recognizes provider-specific Claude names and a declared base model for opaque deployments", () => { + const models = resolve( + deployment("small", "vertex_ai/claude-haiku-4-5@20251001"), + deployment("large", "bedrock/arn:example:inference-profile/us.anthropic.claude-opus-5-5-v1:0"), + deployment("middle", "azure/private-alias", { + litellm_params: { model: "azure/private-alias" }, + model_info: { base_model: "gpt-6.1-sol" }, + }), + ); + expect(models.map(({ name, family, provider }) => ({ name, family, provider }))).toEqual([ + { name: "large", family: "claude", provider: "Bedrock" }, + { name: "middle", family: "openai", provider: "Azure" }, + { name: "small", family: "claude", provider: "Vertex AI" }, ]); }); - it("returns null when none of the available models are recommended", () => { - const preferred: PreferredTierModels = { - SIMPLE: ["missing-simple"], - MEDIUM: ["missing-medium"], - COMPLEX: ["missing-complex"], - REASONING: ["missing-reasoning"], - }; - - expect(buildAutomaticRouterConfig(models("unknown-model"), [], preferred)).toBeNull(); + it("uses only caller-visible chat models and excludes blocked and recursive targets", () => { + const rows = [ + deployment("hidden", "anthropic/claude-opus-6"), + deployment("blocked", "anthropic/claude-opus-5", { model_info: { blocked: true } }), + deployment("embedding", "anthropic/claude-opus-5"), + deployment("recursive", "auto_router/complexity_router"), + deployment("ready", "anthropic/claude-sonnet-5"), + ]; + expect( + deployedRouterModels( + [ + { model_group: "blocked", mode: "chat" }, + { model_group: "embedding", mode: "embedding" }, + { model_group: "recursive" }, + { model_group: "ready", mode: "chat" }, + ], + rows, + ).map((model) => model.name), + ).toEqual(["ready"]); }); - it("ignores non-chat models and existing auto routers", () => { - const preferred: PreferredTierModels = { - SIMPLE: ["gpt-4o-mini", "smart-router"], - MEDIUM: [], - COMPLEX: [], - REASONING: [], - }; - const available = [ - { model_group: "gpt-4o-mini", mode: "chat" }, - { model_group: "image-model", mode: "image_generation" }, - { model_group: "smart-router", mode: "chat" }, + it("does not claim a mixed-version or unknown model group is a known version", () => { + const rows = [ + deployment("mixed", "anthropic/claude-opus-5"), + deployment("mixed", "anthropic/claude-opus-5-5"), + deployment("claude-sonnet-5", "openai/custom-model"), + deployment("wildcard", "openai/*"), ]; + const models = deployedRouterModels( + [ + { model_group: "mixed", mode: "chat" }, + { model_group: "claude-sonnet-5", mode: "chat" }, + { model_group: "wildcard", mode: "chat" }, + ], + rows, + ); + expect(models.map((model) => model.name)).toEqual(["claude-sonnet-5", "mixed"]); + expect(models.every((model) => model.family === undefined)).toBe(true); + expect(recommendSolverPair(models, "balanced", "auto")).toBeNull(); + }); + it("keeps homogeneous replicas and ignores blocked replicas when resolving identity", () => { + const rows = [ + deployment("group", "anthropic/claude-opus-5-5"), + deployment("group", "bedrock/anthropic.claude-opus-5-5-v1:0"), + deployment("group", "anthropic/claude-opus-5", { model_info: { blocked: true } }), + ]; + const models = deployedRouterModels([{ model_group: "group" }], rows); + expect(models).toHaveLength(1); + expect(models[0]).toMatchObject({ name: "group", version: [5, 5], provider: "Anthropic, Bedrock" }); + }); + + it("keeps arbitrary canonical names available without pretending an unversioned alias is latest", () => { expect( - tierModels( - buildAutomaticRouterConfig(available, [deployment("smart-router", "auto_router/complexity_router")], preferred), + deployedRouterModels([{ model_group: "claude-opus-8-12" }, { model_group: "claude-opus-latest" }], []).map( + (model) => model.name, ), - ).toEqual(["gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini"]); + ).toEqual(["claude-opus-8-12", "claude-opus-latest"]); }); }); -describe("prefillEmptyTiers", () => { - const empty: ComplexityRouterConfigValue = { - classifier_type: "jev", - jev_classifier_config: { model: "jev-custom", timeout_ms: 4200 }, - classification_mode: "user_turn", - tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, - }; - const automatic: ComplexityRouterConfigValue = { - classifier_type: "heuristic_v2", - tiers: { SIMPLE: ["fast"], MEDIUM: ["mid"], COMPLEX: ["strong"], REASONING: ["strong"] }, - tier_model_params: { REASONING: { strong: { reasoning_effort: "high" } } }, - }; - it("fills tiers while preserving a classifier configured before models load", () => { - expect(prefillEmptyTiers(empty, automatic)).toEqual({ - ...empty, - tiers: automatic.tiers, - tier_model_params: automatic.tier_model_params, +describe("recommendSolverPair", () => { + it.each([ + ["cheapest", ["small", "medium"]], + ["balanced", ["small", "large"]], + ["quality", ["medium", "large"]], + ] as const)("uses the requested %s pair", (goal, expected) => { + expect(names(recommendSolverPair(ladder(), goal, "claude"))).toEqual(expected); + }); + + it("selects future deployed releases numerically within each line, without a version allowlist", () => { + const models = resolve( + deployment("old-small", "anthropic/claude-haiku-8-9"), + deployment("new-small", "bedrock/anthropic.claude-haiku-8-10-v1:0"), + deployment("old-large", "anthropic/claude-opus-8-9"), + deployment("new-large", "anthropic/claude-opus-9"), + ); + expect(names(recommendSolverPair(models, "balanced", "claude"))).toEqual(["new-small", "new-large"]); + }); + + it("orders snapshots within a version without confusing a snapshot with the next version", () => { + const models = resolve( + deployment("old", "anthropic/claude-haiku-4-5-20251001"), + deployment("snapshot", "anthropic/claude-haiku-4-5-20251101"), + deployment("next", "anthropic/claude-haiku-4-6"), + deployment("large", "anthropic/claude-opus-5"), + ); + expect(names(recommendSolverPair(models, "balanced", "claude"))).toEqual(["next", "large"]); + expect( + names( + recommendSolverPair( + models.filter((model) => model.name !== "next"), + "balanced", + "claude", + ), + ), + ).toEqual(["snapshot", "large"]); + }); + + it("prefers rolling aliases over snapshots of the same release", () => { + const models = resolve( + deployment("pin", "anthropic/claude-haiku-4-5-20251101"), + deployment("alias", "anthropic/claude-haiku-4-5"), + deployment("hard", "anthropic/claude-opus-5"), + deployment("luna-pin", "openai/gpt-6-luna-2026-01-01"), + deployment("luna", "openai/gpt-6-luna"), + deployment("astra", "openai/gpt-6-astra"), + ); + expect(names(recommendSolverPair(models, "balanced", "claude"))).toEqual(["alias", "hard"]); + expect(names(recommendSolverPair(models, "balanced", "openai"))).toEqual(["luna", "astra"]); + }); + + it.each([ + { models: ["grok-4.20", "grok-4.20-0309"], expected: "grok-4.20" }, + { models: ["grok-4.20-0309", "grok-4.20-0410"], expected: "grok-4.20-0410" }, + { models: ["grok-4.20-0309", "grok-4.21"], expected: "grok-4.21" }, + ])("orders Grok releases and short-date snapshots: $expected", ({ models, expected }) => { + const available = resolve( + deployment("everyday", "openai/gpt-6-luna"), + ...models.map((model) => deployment(model, `xai/${model}`)), + ); + expect(names(recommendSolverPair(available, "balanced", "grok"))).toEqual(["everyday", expected]); + }); + + it("does not rank a major-release snapshot above a newer minor release", () => { + const models = resolve( + deployment("old", "anthropic/claude-5-haiku-20260101"), + deployment("next", "anthropic/claude-haiku-5-1"), + deployment("hard", "anthropic/claude-opus-5"), + ); + expect(names(recommendSolverPair(models, "balanced", "claude"))).toEqual(["next", "hard"]); + }); + + it("prefers a same-family substitute over a matching rank from another family", () => { + const models = [ + ...ladder().filter((model) => model.line !== "haiku"), + ...resolve(deployment("luna", "openai/gpt-6-luna")), + ]; + const pair = recommendSolverPair(models, "balanced", "claude"); + expect(names(pair)).toEqual(["medium", "large"]); + expect(pair?.notice).toContain("closest available pair"); + }); + + it("prefers a complete family for Auto and explains a cross-family substitute when explicitly requested", () => { + const models = [ + ...ladder().filter((model) => model.line === "sonnet"), + ...resolve( + deployment("luna", "bedrock/us.openai.gpt-6-luna"), + deployment("astra", "bedrock/us.openai.gpt-6-astra"), + ), + ]; + expect(names(recommendSolverPair(models, "balanced", "auto"))).toEqual(["luna", "astra"]); + const selected = recommendSolverPair(models, "quality", "claude"); + expect(names(selected)).toEqual(["medium", "astra"]); + expect(selected?.notice).toContain("two model families"); + }); + + it.each([ + ["glm", "zai/glm-7.3-flash", "zai/glm-7.3"], + ["deepseek", "deepseek/deepseek-v9-flash", "deepseek/deepseek-v9-pro"], + ] as const)("handles the %s two-line family for every goal", (family, small, large) => { + const models = resolve(deployment("small", small), deployment("large", large)); + for (const goal of ["cheapest", "balanced", "quality"] as const) + expect(names(recommendSolverPair(models, goal, family))).toEqual(["small", "large"]); + }); + + it("does not fabricate a second solver or substitute a different family when none of the requested family exists", () => { + expect(recommendSolverPair(ladder().slice(0, 1), "balanced", "auto")).toBeNull(); + expect(recommendSolverPair(ladder(), "balanced", "openai")).toBeNull(); + expect(recommendSolverPair([], "balanced", "auto")).toBeNull(); + }); + + it("retains Gemini family suggestions and recognizes the existing Grok fallback", () => { + const models = resolve( + deployment("lite", "gemini/gemini-3.5-flash-lite"), + deployment("flash-old", "vertex_ai/gemini-3.5-flash"), + deployment("flash", "vertex_ai/gemini-3.8-flash"), + deployment("pro", "gemini/gemini-3.1-pro-preview"), + deployment("grok", "xai/grok-4.6"), + ); + expect(names(recommendSolverPair(models, "cheapest", "gemini"))).toEqual(["lite", "flash"]); + expect(names(recommendSolverPair(models, "balanced", "gemini"))).toEqual(["lite", "pro"]); + expect(names(recommendSolverPair(models, "quality", "gemini"))).toEqual(["flash", "pro"]); + expect(models.find((m) => m.name === "grok")).toMatchObject({ family: "grok", provider: "xAI" }); + expect(names(recommendSolverPair(models, "balanced", "grok"))).toEqual(["lite", "grok"]); + }); + + it("uses the latest deployed Terra as a fallback without replacing the preferred OpenAI pairs", () => { + const models = resolve( + deployment("terra-old", "openai/gpt-5.6-terra"), + deployment("terra", "openai/gpt-6.1-terra"), + deployment("sol", "openai/gpt-6.1-sol"), + ); + expect(names(recommendSolverPair(models, "balanced", "openai"))).toEqual(["terra", "sol"]); + const complete = [ + ...models, + ...resolve(deployment("luna", "openai/gpt-6-luna"), deployment("astra", "openai/gpt-6-astra")), + ]; + expect(names(recommendSolverPair(complete, "cheapest", "openai"))).toEqual(["luna", "sol"]); + expect(names(recommendSolverPair(complete, "balanced", "openai"))).toEqual(["luna", "astra"]); + expect(names(recommendSolverPair(complete, "quality", "openai"))).toEqual(["sol", "astra"]); + }); + + it("suggests a deterministic equal-rank cross-family fallback only when no ordered pair exists", () => { + const models = resolve(deployment("opus", "anthropic/claude-opus-5-5"), deployment("astra", "openai/gpt-6-astra")); + const pair = recommendSolverPair(models, "balanced", "auto"); + expect(names(pair)).toEqual(["astra", "opus"]); + expect(pair?.notice).toContain("two model families"); + expect(names(recommendSolverPair([...models].reverse(), "balanced", "auto"))).toEqual(names(pair)); + const withLuna = [...models, ...resolve(deployment("luna", "openai/gpt-6-luna"))]; + expect(names(recommendSolverPair(withLuna, "balanced", "auto"))).toEqual(["luna", "astra"]); + }); + + it("does not prefer native-provider deployments over a newer cloud deployment", () => { + const models = resolve( + deployment("native", "openai/gpt-6-luna"), + deployment("cloud", "bedrock/global.openai.gpt-6.2-luna"), + deployment("strong", "bedrock/global.openai.gpt-6-astra"), + ); + expect(names(recommendSolverPair(models, "balanced", "openai"))).toEqual(["cloud", "strong"]); + }); +}); + +describe("solverPairConfig", () => { + it("builds two solver references using built-in rules without paid classifiers, custom tiers or output overrides", () => { + expect(solverPairConfig("everyday", "hard")).toEqual({ + classifier_type: "heuristic", + classification_mode: "user_turn", + tiers: { SIMPLE: ["everyday"], MEDIUM: ["everyday"], COMPLEX: ["hard"], REASONING: ["hard"] }, }); }); - it("leaves the draft alone when no recommended models are available", () => { - expect(prefillEmptyTiers(empty, null)).toBe(empty); +}); + +describe("simple setup validation", () => { + it("avoids existing names without requiring a naming decision", () => { + expect(nextRouterName([])).toBe("auto-router"); + expect(nextRouterName(["auto-router", "auto-router-2", "auto-router-4"])).toBe("auto-router-3"); }); - it("preserves a partial manual assignment", () => { - const partial = { ...empty, tiers: { ...empty.tiers, SIMPLE: ["manual"] } }; - expect(prefillEmptyTiers(partial, automatic)).toBe(partial); - }); - it("preserves a custom tier set created before models load", () => { - const custom = { ...empty, custom_tier_set: { tiers: [], fallback_tier_id: "" } }; - expect(prefillEmptyTiers(custom, automatic)).toBe(custom); - }); - it.each(["capability", "llm_v2"] as const)("does not prefill %s solver pools", (classifier_type) => { - const solver = { ...empty, classifier_type }; - expect(prefillEmptyTiers(solver, automatic)).toBe(solver); + it("blocks stale or incomplete inventory and same-line selections", () => { + const [small, large] = [ladder()[0], ladder()[2]]; + expect(simpleSetupError(true, false, small, large)).not.toBeNull(); + expect(simpleSetupError(false, true, small, large)).not.toBeNull(); + expect(simpleSetupError(false, false, small)).not.toBeNull(); + expect(simpleSetupError(false, false, small, small)).not.toBeNull(); + expect(simpleSetupError(false, false, small, large)).toBeNull(); }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts index 8b60336cd27..39476470928 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.ts @@ -1,110 +1,263 @@ -import { activeTierRows } from "./tier_rows"; -import { isForecastClassifier } from "./forecast_classifier_config"; -import { isAutoRouterDeployment, type AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; +import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; +import { isAutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; import type { ModelGroup } from "@/components/llm_calls/fetch_models"; -import { resolveAvailableModel, type AutoRouterPreset, type ModelAvailability } from "@/lib/autorouter_presets"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; -const TIER_NAMES = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const; -type TierName = (typeof TIER_NAMES)[number]; -export type PreferredTierModels = Record; +export const ROUTER_FAMILIES = { + claude: "Claude", + openai: "OpenAI", + gemini: "Gemini", + grok: "Grok", + glm: "GLM", + deepseek: "DeepSeek", +} as const; +export const ROUTER_GOALS = { cheapest: "Cheapest", balanced: "Balanced", quality: "Best quality" } as const; +export type RouterFamily = keyof typeof ROUTER_FAMILIES; +export type RouterGoal = keyof typeof ROUTER_GOALS; +export type FamilyPreference = RouterFamily | "auto"; -const REASONING_EFFORT_STRENGTH = ["max", "xhigh", "high", "medium", "low", "minimal", "none"] as const; +type ModelIdentity = { + family: RouterFamily; + line: string; + rank: number; + version: readonly number[]; + snapshot: number; + model: string; +}; +export type SolverModel = Partial & { name: string; provider: string; model: string }; +export type RouterModel = ModelIdentity & SolverModel; +export type SolverPair = { everyday: RouterModel; hard: RouterModel; notice: string | null }; -const CURRENT_TIER_MODELS: PreferredTierModels = { - SIMPLE: ["gpt-5.6-luna", "claude-haiku-4-5", "gemini-3.5-flash-lite", "deepseek-v4-flash"], - MEDIUM: ["gpt-5.6-terra", "claude-sonnet-5", "gemini-3.8-flash", "deepseek-v4-flash"], - COMPLEX: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"], - REASONING: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"], +const releaseVersion = (raw: string, date?: string) => { + const snapshot = raw.match(/[-.](\d{8})$/); + const release = snapshot ? raw.slice(0, -9) : raw; + return { + version: release.split(/[.-]/).map(Number), + // An undated alias follows the current snapshot of its release. + snapshot: date || snapshot ? Number((date ?? snapshot![1]).replaceAll("-", "")) : Number.MAX_SAFE_INTEGER, + }; }; -export const buildPreferredTierModels = ( - presets: AutoRouterPreset[], - availability: ModelAvailability, -): PreferredTierModels => - Object.fromEntries( - TIER_NAMES.map((tier) => [ - tier, - Array.from( - new Set( - [ - ...CURRENT_TIER_MODELS[tier], - ...presets.flatMap((preset) => preset.complexity_router_config.tiers[tier]), - ].flatMap((model) => { - const resolved = resolveAvailableModel(model, availability); - return resolved ? [resolved] : []; - }), - ), +const parseModel = (raw: string): ModelIdentity | null => { + const model = raw + .toLowerCase() + .split("/") + .at(-1)! + .replace(/^(?:us|eu|apac|global|au|jp|us-gov)\./, "") + .replace(/^(?:anthropic|openai|zai|deepseek)\./, "") + .replace(/^databricks-/, "") + .replace(/@default$/, "") + .replace(/-v\d+(?::\d+)?$/, ""); + const versionFirst = model.match(/^claude-(\d+(?:[.-]\d+)*)-(haiku|sonnet|opus)(?:[-@]([0-9]{8}))?$/); + const claude = + model.match(/^claude-(haiku|sonnet|opus)-(\d+(?:[.-]\d+)*)(?:@([0-9]+))?$/) ?? + (versionFirst ? [versionFirst[0], versionFirst[2], versionFirst[1], versionFirst[3]] : null); + if (claude) + return { + family: "claude", + line: claude[1], + rank: ["haiku", "sonnet", "opus"].indexOf(claude[1]), + ...releaseVersion(claude[2], claude[3]), + model, + }; + const openai = model.match(/^gpt-(\d+(?:[.-]\d+)*)-(luna|terra|sol|astra)(?:-(\d{4}-\d{2}-\d{2}))?$/); + if (openai) + return { + family: "openai", + line: openai[2], + rank: openai[2] === "terra" ? 0.5 : ["luna", "sol", "astra"].indexOf(openai[2]), + ...releaseVersion(openai[1], openai[3]), + model, + }; + const gemini = model.match(/^gemini-(\d+(?:[.-]\d+)*)-(flash-lite|flash|pro)(?:-preview)?$/); + if (gemini) + return { + family: "gemini", + line: gemini[2], + rank: ["flash-lite", "flash", "pro"].indexOf(gemini[2]), + ...releaseVersion(gemini[1]), + model, + }; + const grok = model.match(/^grok-(\d+(?:[.-]\d+)*?)(?:-(\d{4}))?$/); + if (grok) return { family: "grok", line: "grok", rank: 2, ...releaseVersion(grok[1], grok[2]), model }; + const glm = model.match(/^glm-(\d+(?:[.-]\d+)*)(-flash)?$/); + if (glm) + return { + family: "glm", + line: glm[2] ? "flash" : "glm", + rank: glm[2] ? 0 : 2, + ...releaseVersion(glm[1]), + model, + }; + const deepseek = model.match(/^deepseek-v(\d+(?:[.-]\d+)*)-(flash|pro)$/); + if (deepseek) + return { + family: "deepseek", + line: deepseek[2], + rank: deepseek[2] === "flash" ? 0 : 2, + ...releaseVersion(deepseek[1]), + model, + }; + return null; +}; + +const identityOf = (deployment: AutoRouterDeployment): ModelIdentity | null => + parseModel(deployment.litellm_params?.model ?? "") ?? + parseModel(deployment.model_info?.base_model ?? deployment.litellm_params?.base_model ?? ""); + +const providerOf = (deployment: AutoRouterDeployment, group: ModelGroup, family?: RouterFamily): string => { + const raw = deployment.litellm_params?.model ?? ""; + const provider = + deployment.litellm_params?.custom_llm_provider || + (raw.includes("/") ? raw.split("/")[0] : group.providers?.join(", ")) || + (family + ? { claude: "anthropic", glm: "zai", openai: "openai", gemini: "gemini", grok: "xai", deepseek: "deepseek" }[ + family + ] + : ""); + if (provider.startsWith("bedrock")) return "Bedrock"; + return ( + ( + { + anthropic: "Anthropic", + openai: "OpenAI", + gemini: "Google AI", + xai: "xAI", + azure: "Azure", + azure_ai: "Azure AI", + databricks: "Databricks", + github_copilot: "GitHub Copilot", + "vertex_ai-anthropic_models": "Vertex AI", + vertex_ai: "Vertex AI", + zai: "Z.ai", + deepseek: "DeepSeek", + } as Record + )[provider] ?? provider + ); +}; + +const compareVersion = (left: readonly number[], right: readonly number[]): number => { + const index = Array.from({ length: Math.max(left.length, right.length) }, (_, i) => i).find( + (i) => (left[i] ?? 0) !== (right[i] ?? 0), + ); + return index === undefined ? 0 : (left[index] ?? 0) - (right[index] ?? 0); +}; + +const compareRelease = (left: ModelIdentity, right: ModelIdentity): number => + compareVersion(left.version, right.version) || left.snapshot - right.snapshot; + +export const deployedRouterModels = ( + groups: readonly ModelGroup[], + deployments: readonly AutoRouterDeployment[], +): SolverModel[] => + groups + .flatMap((group) => { + if (group.mode && group.mode !== "chat") return []; + const rows = deployments.filter((row) => row.model_name === group.model_group); + if ( + rows.some(isAutoRouterDeployment) || + group.model_group.includes("*") || + group.model_group.startsWith("auto_router/") + ) + return []; + const active = rows.filter((row) => !row.model_info?.blocked); + if (rows.length > 0 && active.length === 0) return []; + if (active.some((row) => row.model_info?.mode && row.model_info.mode !== "chat")) return []; + if (active.some((row) => row.litellm_params?.model?.includes("*"))) return []; + const identities = active.length > 0 ? active.map(identityOf) : [parseModel(group.model_group)]; + const first = identities[0]; + const homogeneous = + first && + identities.every((identity) => { + if (!identity) return false; + const sameLine = identity.family === first.family && identity.line === first.line; + return sameLine && compareRelease(identity, first) === 0; + }); + const identity = homogeneous ? first : undefined; + const providers = + active.length > 0 + ? active.map((row) => providerOf(row, group, identity?.family)) + : [providerOf({}, group, identity?.family)]; + return [ + { + ...identity, + name: group.model_group, + model: identity?.model ?? group.model_group, + provider: [...new Set(providers)].toSorted().join(", "), + }, + ]; + }) + .toSorted((a, b) => a.name.localeCompare(b.name)); + +const latestLines = (models: readonly RouterModel[]): RouterModel[] => + models.filter( + (model) => + !models.some( + (other) => other.family === model.family && other.line === model.line && compareRelease(other, model) > 0, ), - ]), - ) as PreferredTierModels; - -const selectPreferredTierModels = ( - preferredByTier: PreferredTierModels, - usableNames: ReadonlySet, -): [string, string, string, string] | null => { - const preferred = TIER_NAMES.map((tier) => preferredByTier[tier].find((name) => usableNames.has(name))); - const candidates = preferred.flatMap((model, tier) => (model ? [{ model, tier }] : [])); - if (candidates.length === 0) return null; - - const nearest = (tier: number): string => - [...candidates].sort( - (left, right) => Math.abs(left.tier - tier) - Math.abs(right.tier - tier) || left.tier - right.tier, - )[0].model; - return preferred.map((model, tier) => model ?? nearest(tier)) as [string, string, string, string]; -}; - -export const buildAutomaticRouterConfig = ( - models: ModelGroup[], - deployments: AutoRouterDeployment[], - preferredByTier: PreferredTierModels, -): ComplexityRouterConfigValue | null => { - const autoRouterNames: ReadonlySet = new Set( - deployments - .filter(isAutoRouterDeployment) - .flatMap((deployment) => (deployment.model_name ? [deployment.model_name] : [])), ); - const names = Array.from( - new Set( - models - .filter((model) => model.mode === undefined || model.mode === "chat") - .map((model) => model.model_group) - .filter((name) => name && !name.startsWith("auto_router/") && !autoRouterNames.has(name)), - ), + +const requestedRanks = (goal: RouterGoal): readonly [number, number] => + (({ cheapest: [0, 1], quality: [1, 2], balanced: [0, 2] }) as const)[goal]; + +export const recommendSolverPair = ( + models: readonly SolverModel[], + goal: RouterGoal, + family: FamilyPreference, +): SolverPair | null => { + const recognized = models.filter((model): model is RouterModel => model.family !== undefined); + const latest = latestLines(recognized); + const [low, high] = requestedRanks(goal); + const pairs = latest.flatMap((everyday) => + latest + .filter((hard) => { + const matchesFamily = family === "auto" || everyday.family === family || hard.family === family; + const ordered = hard.rank > everyday.rank; + const equalRankFallback = hard.rank === everyday.rank && hard.family !== everyday.family; + return (ordered || equalRankFallback) && matchesFamily; + }) + .map((hard) => ({ everyday, hard })), ); - if (names.length === 0) return null; - const usableNames: ReadonlySet = new Set(names); - const selected = selectPreferredTierModels(preferredByTier, usableNames); - if (selected === null) return null; - - const supportedReasoningEfforts = models.find( - (model) => model.model_group === selected[3], - )?.supported_reasoning_efforts; - const reasoningEffort = REASONING_EFFORT_STRENGTH.find((effort) => supportedReasoningEfforts?.includes(effort)); - - return { - tiers: { - SIMPLE: [selected[0]], - MEDIUM: [selected[1]], - COMPLEX: [selected[2]], - REASONING: [selected[3]], - }, - classifier_type: "heuristic_v2", - ...(reasoningEffort && { - tier_model_params: { REASONING: { [selected[3]]: { reasoning_effort: reasoningEffort } } }, - }), - }; + const score = ({ everyday, hard }: { everyday: RouterModel; hard: RouterModel }): number => + (everyday.family === hard.family ? 0 : 100) + + (everyday.rank === hard.rank ? 10 : 0) + + Math.abs(everyday.rank - low) + + Math.abs(hard.rank - high); + const chosen = pairs.toSorted( + (a, b) => + score(a) - score(b) || a.everyday.name.localeCompare(b.everyday.name) || a.hard.name.localeCompare(b.hard.name), + )[0]; + if (!chosen) return null; + const mixed = chosen.everyday.family !== chosen.hard.family; + const substituted = chosen.everyday.rank !== low || chosen.hard.rank !== high; + const mixedNotice = mixed ? "A complete pair is unavailable in one family. Using two model families." : null; + const substituteNotice = substituted ? "Using the closest available pair in this family." : null; + return { ...chosen, notice: mixedNotice ?? substituteNotice }; }; -export const prefillEmptyTiers = ( - current: ComplexityRouterConfigValue, - automatic: ComplexityRouterConfigValue | null, -): ComplexityRouterConfigValue => { - if (!automatic || current.custom_tier_set || isForecastClassifier(current.classifier_type)) return current; - if (activeTierRows(current).some((row) => row.models.length > 0)) return current; - return { - ...current, - tiers: { ...current.tiers, ...automatic.tiers }, - tier_model_params: { ...current.tier_model_params, ...automatic.tier_model_params }, - }; +export const solverPairConfig = (everyday: string, hard: string): ComplexityRouterConfigValue => ({ + classifier_type: "heuristic", + classification_mode: "user_turn", + tiers: { SIMPLE: [everyday], MEDIUM: [everyday], COMPLEX: [hard], REASONING: [hard] }, +}); + +export const simpleSetupError = ( + loading: boolean, + failed: boolean, + everyday?: SolverModel, + hard?: SolverModel, +): string | null => { + if (loading) return "Loading deployed models"; + if (failed) return "Could not verify deployed models"; + if (!everyday || !hard) return "Select two different deployed models"; + if (everyday.name === hard.name) return "Select two different deployed models"; + return null; +}; + +export const nextRouterName = (names: readonly string[]): string => { + const candidates = [ + "auto-router", + ...Array.from({ length: names.length + 1 }, (_, index) => `auto-router-${index + 2}`), + ]; + return candidates.find((name) => !names.includes(name))!; };