Merge pull request #39693 from BerriAI/litellm_auto_setup_simple

feat(ui): add one-click Auto Router setup
This commit is contained in:
moe-berri 2026-09-03 20:28:01 -07:00 • committed by GitHub
commit 04c6ce9ac3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 444 additions and 69 deletions

View file

@ -19,7 +19,6 @@ vi.mock(
"@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets",
async () => await import("../../../tests/mocks/autoRouterPresets"),
);
const getAllPresets = (): AutoRouterPreset[] => BUNDLED_PRESETS;
const getPresetByKey = (key: string): AutoRouterPreset | undefined => BUNDLED_PRESETS.find((p) => p.key === key);
@ -155,6 +154,58 @@ describe("AddAutoRouterTab", () => {
expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument();
});
it("hides automatic setup when no available model is recommended", async () => {
mockFetchAvailableModels.mockResolvedValue([
{ model_group: "unknown-model-a", mode: "chat" },
{ model_group: "unknown-model-b", mode: "chat" },
]);
renderWithProviders(<Harness />);
openTemplateDropdown();
await waitFor(() => expect(optionByLabel("Anthropic Family")).toHaveTextContent("Missing:"));
expect(screen.queryByTestId("configure-automatically-button")).not.toBeInTheDocument();
});
it("mixes preferred tier models even when one complete preset is available", async () => {
const anthropicPreset = getPresetByKey("anthropic_family")!;
mockFetchAvailableModels.mockResolvedValue(
[...getRequiredModelsInPreset(anthropicPreset), "gpt-5.6-luna"].map((model_group) => ({
model_group,
mode: "chat",
})),
);
mockFetchAllModelDeployments.mockResolvedValue([]);
renderWithProviders(<Harness />);
const button = await screen.findByTestId("configure-automatically-button");
await userEvent.click(button);
expect(
screen.getByText(
/Simple: gpt-5.6-luna.*Medium: claude-sonnet-5.*Complex: claude-opus-5.*Reasoning: claude-opus-5/,
),
).toBeInTheDocument();
expect(toast.success).not.toHaveBeenCalledWith(expect.stringContaining("Configured with"));
});
it("mixes available models from the preferred tier catalog when no complete template fits", async () => {
mockFetchAvailableModels.mockResolvedValue(
["gpt-5.6-luna", "claude-sonnet-5", "gpt-5.6-sol"].map((model_group) => ({
model_group,
mode: "chat",
})),
);
mockFetchAllModelDeployments.mockResolvedValue([]);
renderWithProviders(<Harness />);
const button = await screen.findByTestId("configure-automatically-button");
await userEvent.click(button);
expect(
screen.getByText(/Simple: gpt-5.6-luna.*Medium: claude-sonnet-5.*Complex: gpt-5.6-sol.*Reasoning: gpt-5.6-sol/),
).toBeInTheDocument();
});
// Nothing is filled in, so there is nothing to submit. The button reports that itself instead of
// accepting a click and answering with a toast.
it("offers no submit at all until every tier has a model", async () => {

View file

@ -64,6 +64,7 @@ import {
} from "@/lib/autorouter_presets";
import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets";
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
import { buildAutomaticRouterConfig, buildPreferredTierModels } from "./auto_setup";
interface AddAutoRouterTabProps {
handleOk: () => void;
@ -242,6 +243,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
refetch: refetchPresets,
} = useAutoRouterPresets();
const presets = presetsData ?? NO_PRESETS;
const automaticSetupLoading = modelsLoading || presetsPending;
const presetsUnavailable = presetsError && presetsData === undefined;
// react-query keeps the last successful list around when a later refetch fails, so isError alone
// can't tell "never loaded" apart from "loaded, then a background refetch errored" - only the
@ -266,6 +268,14 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
),
[modelInfo],
);
const preferredTierModels = React.useMemo(
() => buildPreferredTierModels(presets, availability),
[presets, availability],
);
const automaticRouterConfig = React.useMemo(
() => buildAutomaticRouterConfig(modelInfo, deployments ?? [], preferredTierModels),
[modelInfo, deployments, preferredTierModels],
);
// A preset's models can only be trusted against a successfully loaded list. Selection and the
// greyed-out state derive from this one function, so a preset that cannot be selected can never
@ -313,6 +323,14 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
setEscalationKeywords(prefill.escalationKeywords);
};
const handleAutomaticSetup = () => {
if (automaticRouterConfig === null) return;
setSelectedPreset(undefined);
applyPrefill({ ...buildEmptyPrefill(), complexityRouterConfig: automaticRouterConfig });
setDetailsExpanded(false);
toast.success("Automatic setup created", { description: tierConfigSummary(automaticRouterConfig) });
};
const handlePresetChange = (presetKey: string | undefined) => {
if (!presetKey || presetKey === "custom") {
setSelectedPreset(presetKey);
@ -500,77 +518,92 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
<CardContent>
<form onSubmit={form.handleSubmit(() => handleAutoRouterSubmit())} noValidate>
<FieldGroup>
<FormField
control={form.control}
name="auto_router_name"
label={labelWithHint("Auto Router Name", "Unique name for this auto router configuration")}
>
{({ ref, ...field }) => <Input {...field} ref={ref} placeholder="e.g., smart_router, auto_router_1" />}
</FormField>
<div>
<label className="block text-sm font-medium text-foreground mb-2">Template</label>
<Select
items={templateItems}
value={selectedPreset ?? null}
onValueChange={(presetKey: string | null) => handlePresetChange(presetKey ?? undefined)}
<FormField
control={form.control}
name="auto_router_name"
label={labelWithHint("Auto Router Name", "Unique name for this auto router configuration")}
>
<SelectTrigger data-testid="template-selector" className="w-full">
<SelectValue placeholder="Choose a template or select Custom to define your own" />
</SelectTrigger>
<SelectContent>
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
const disabledHint = presetDisabledHint(presetState);
const hintClass = isPresetHintAlarming(presetState)
? "text-destructive"
: "text-muted-foreground";
const matchedHint =
presetState.kind === "available" && presetState.viaDeployments
? "Matches your deployments"
: null;
{({ ref, ...field }) => (
<Input {...field} ref={ref} placeholder="e.g., smart_router, auto_router_1" />
)}
</FormField>
return (
<SelectItem
key={preset.key}
value={preset.key}
label={preset.label}
disabled={disabledHint !== null}
title={disabledHint ?? preset.description}
>
<div>
<div className="font-medium">{preset.label}</div>
<div className="text-xs text-muted-foreground">{preset.description}</div>
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
{matchedHint && <div className="text-xs mt-1 text-success">{matchedHint}</div>}
</div>
</SelectItem>
);
})}
<SelectItem value="custom" label="Custom Configuration">
<div>
<div className="font-medium">Custom Configuration</div>
<div className="text-xs text-muted-foreground">Define your auto router from scratch</div>
</div>
</SelectItem>
</SelectContent>
</Select>
{modelsUnverifiable && (
<div className="text-xs mt-1 text-destructive">
Could not load available models.{" "}
<button type="button" className="underline" onClick={() => refetchModels()}>
Retry
</button>
</div>
)}
{presetsPending && <div className="text-xs mt-1 text-muted-foreground">Loading templates...</div>}
{presetsUnavailable && (
<div className="text-xs mt-1 text-destructive">
Could not load templates, so only Custom Configuration is shown.{" "}
<button type="button" className="underline" onClick={() => void refetchPresets()}>
Retry
</button>
</div>
{!automaticSetupLoading && automaticRouterConfig && (
<button
type="button"
className="mt-3 rounded-sm text-sm font-medium text-blue-600 hover:text-blue-700 hover:underline focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-blue-500 focus-visible:ring-offset-2"
data-testid="configure-automatically-button"
onClick={handleAutomaticSetup}
>
Configure automatically
</button>
)}
<div className="mt-5">
<label className="block text-sm font-medium text-foreground mb-2">Template</label>
<Select
items={templateItems}
value={selectedPreset ?? null}
onValueChange={(presetKey: string | null) => handlePresetChange(presetKey ?? undefined)}
>
<SelectTrigger data-testid="template-selector" className="w-full">
<SelectValue placeholder="Choose a template or select Custom to define your own" />
</SelectTrigger>
<SelectContent>
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
const disabledHint = presetDisabledHint(presetState);
const hintClass = isPresetHintAlarming(presetState)
? "text-destructive"
: "text-muted-foreground";
const matchedHint =
presetState.kind === "available" && presetState.viaDeployments
? "Matches your deployments"
: null;
return (
<SelectItem
key={preset.key}
value={preset.key}
label={preset.label}
disabled={disabledHint !== null}
title={disabledHint ?? preset.description}
>
<div>
<div className="font-medium">{preset.label}</div>
<div className="text-xs text-muted-foreground">{preset.description}</div>
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
{matchedHint && <div className="text-xs mt-1 text-success">{matchedHint}</div>}
</div>
</SelectItem>
);
})}
<SelectItem value="custom" label="Custom Configuration">
<div>
<div className="font-medium">Custom Configuration</div>
<div className="text-xs text-muted-foreground">Define your auto router from scratch</div>
</div>
</SelectItem>
</SelectContent>
</Select>
{modelsUnverifiable && (
<div className="text-xs mt-1 text-destructive">
Could not load available models.{" "}
<button type="button" className="underline" onClick={() => refetchModels()}>
Retry
</button>
</div>
)}
{presetsPending && <div className="text-xs mt-1 text-muted-foreground">Loading templates...</div>}
{presetsUnavailable && (
<div className="text-xs mt-1 text-destructive">
Could not load templates, so only Custom Configuration is shown.{" "}
<button type="button" className="underline" onClick={() => void refetchPresets()}>
Retry
</button>
</div>
)}
</div>
</div>
{requiresTeamScope && (

View file

@ -0,0 +1,196 @@
import { describe, expect, it } from "vitest";
import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
import { buildModelAvailability } from "@/lib/autorouter_presets";
import { buildAutomaticRouterConfig, buildPreferredTierModels, type PreferredTierModels } from "./auto_setup";
const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" }));
const reasoningModel = (model_group: string, supported_reasoning_efforts: string[]) => ({
model_group,
mode: "chat",
supports_reasoning: true,
supported_reasoning_efforts,
});
const deployment = (model_name: string, model = model_name): AutoRouterDeployment => ({
model_name,
litellm_params: { model },
});
const tierModels = (config: ReturnType<typeof buildAutomaticRouterConfig>) =>
config && Object.values(config.tiers).map((tier) => (typeof tier === "string" ? tier : tier[0]));
describe("buildPreferredTierModels", () => {
it("recognizes curated models that are not in a preset", () => {
const available = ["gpt-5.6-luna", "claude-sonnet-5", "grok-4.6", "deepseek-v4-pro"];
const availability = buildModelAvailability(available, []);
const preferred = buildPreferredTierModels([], availability);
const expected: PreferredTierModels = {
SIMPLE: ["gpt-5.6-luna"],
MEDIUM: ["claude-sonnet-5"],
COMPLEX: ["deepseek-v4-pro", "grok-4.6"],
REASONING: ["deepseek-v4-pro", "grok-4.6"],
};
expect(preferred).toEqual(expected);
});
it("prefers the current model ladder over older preset entries", () => {
const availability = buildModelAvailability(["gpt-5.6-luna", "gpt-4o-mini"], []);
const preferred = buildPreferredTierModels(
[
{
key: "old",
label: "Old",
description: "Old model",
complexity_router_config: {
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
classifier_type: "heuristic_v2",
},
},
],
availability,
);
expect(preferred.SIMPLE).toEqual(["gpt-5.6-luna", "gpt-4o-mini"]);
});
});
describe("buildAutomaticRouterConfig", () => {
it("selects one preferred model for each tier", () => {
const preferred: PreferredTierModels = {
SIMPLE: ["simple"],
MEDIUM: ["medium"],
COMPLEX: ["complex"],
REASONING: ["reasoning"],
};
expect(
tierModels(buildAutomaticRouterConfig(models("simple", "medium", "complex", "reasoning"), [], preferred)),
).toEqual(["simple", "medium", "complex", "reasoning"]);
});
it.each([
{
provider: "OpenAI",
available: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra"],
expected: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra", "gpt-6-astra"],
supportedEfforts: ["low", "medium", "high", "xhigh", "max"],
effort: "max",
},
{
provider: "Anthropic",
available: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
expected: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5", "claude-opus-5"],
supportedEfforts: ["low", "medium", "high", "max"],
effort: "max",
},
{
provider: "Google",
available: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview"],
expected: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3.1-pro-preview"],
supportedEfforts: ["low", "medium", "high"],
effort: "high",
},
{
provider: "DeepSeek",
available: ["deepseek-v4-flash", "deepseek-v4-pro"],
expected: ["deepseek-v4-flash", "deepseek-v4-flash", "deepseek-v4-pro", "deepseek-v4-pro"],
supportedEfforts: ["none", "high"],
effort: "high",
},
{
provider: "xAI",
available: ["grok-4.6"],
expected: ["grok-4.6", "grok-4.6", "grok-4.6", "grok-4.6"],
supportedEfforts: ["low", "medium", "high", "xhigh"],
effort: "xhigh",
},
])(
"uses the current $provider ladder and strongest advertised reasoning effort",
({ available, expected, supportedEfforts, effort }) => {
const availability = buildModelAvailability(available, []);
const preferred = buildPreferredTierModels([], availability);
const modelInfo = models(...available).map((model) =>
model.model_group === expected[3] ? reasoningModel(model.model_group, supportedEfforts) : model,
);
const config = buildAutomaticRouterConfig(modelInfo, [], preferred);
expect(tierModels(config)).toEqual(expected);
expect(config?.tier_model_params).toEqual({
REASONING: { [expected[3]]: { reasoning_effort: effort } },
});
},
);
it("never exceeds the selected model group's advertised reasoning efforts", () => {
const available = ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-5.6-sol"];
const availability = buildModelAvailability(available, []);
const preferred = buildPreferredTierModels([], availability);
const modelInfo = [
...models("gpt-5.6-luna", "gpt-5.6-terra"),
reasoningModel("gpt-5.6-sol", ["none", "low", "medium", "high", "xhigh"]),
];
const config = buildAutomaticRouterConfig(modelInfo, [], preferred);
expect(config?.tier_model_params).toEqual({
REASONING: { "gpt-5.6-sol": { reasoning_effort: "xhigh" } },
});
});
it("leaves reasoning effort unset when the proxy does not report supported values", () => {
const available = ["grok-4.6"];
const availability = buildModelAvailability(available, []);
const preferred = buildPreferredTierModels([], availability);
const config = buildAutomaticRouterConfig(models(...available), [], preferred);
expect(config?.tier_model_params).toBeUndefined();
});
it("reuses the closest available tier when a tier has no match", () => {
const preferred: PreferredTierModels = {
SIMPLE: ["simple"],
MEDIUM: [],
COMPLEX: ["complex"],
REASONING: [],
};
expect(tierModels(buildAutomaticRouterConfig(models("simple", "complex"), [], preferred))).toEqual([
"simple",
"simple",
"complex",
"complex",
]);
});
it("returns null when none of the available models are recommended", () => {
const preferred: PreferredTierModels = {
SIMPLE: ["missing-simple"],
MEDIUM: ["missing-medium"],
COMPLEX: ["missing-complex"],
REASONING: ["missing-reasoning"],
};
expect(buildAutomaticRouterConfig(models("unknown-model"), [], preferred)).toBeNull();
});
it("ignores non-chat models and existing auto routers", () => {
const preferred: PreferredTierModels = {
SIMPLE: ["gpt-4o-mini", "smart-router"],
MEDIUM: [],
COMPLEX: [],
REASONING: [],
};
const available = [
{ model_group: "gpt-4o-mini", mode: "chat" },
{ model_group: "image-model", mode: "image_generation" },
{ model_group: "smart-router", mode: "chat" },
];
expect(
tierModels(
buildAutomaticRouterConfig(available, [deployment("smart-router", "auto_router/complexity_router")], preferred),
),
).toEqual(["gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini", "gpt-4o-mini"]);
});
});

View file

@ -0,0 +1,95 @@
import { isAutoRouterDeployment, type AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
import type { ModelGroup } from "@/components/llm_calls/fetch_models";
import { resolveAvailableModel, type AutoRouterPreset, type ModelAvailability } from "@/lib/autorouter_presets";
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
const TIER_NAMES = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const;
type TierName = (typeof TIER_NAMES)[number];
export type PreferredTierModels = Record<TierName, string[]>;
const REASONING_EFFORT_STRENGTH = ["max", "xhigh", "high", "medium", "low", "minimal", "none"] as const;
const CURRENT_TIER_MODELS: PreferredTierModels = {
SIMPLE: ["gpt-5.6-luna", "claude-haiku-4-5", "gemini-3.5-flash-lite", "deepseek-v4-flash"],
MEDIUM: ["gpt-5.6-terra", "claude-sonnet-5", "gemini-3.8-flash", "deepseek-v4-flash"],
COMPLEX: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"],
REASONING: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"],
};
export const buildPreferredTierModels = (
presets: AutoRouterPreset[],
availability: ModelAvailability,
): PreferredTierModels =>
Object.fromEntries(
TIER_NAMES.map((tier) => [
tier,
Array.from(
new Set(
[
...CURRENT_TIER_MODELS[tier],
...presets.flatMap((preset) => preset.complexity_router_config.tiers[tier]),
].flatMap((model) => {
const resolved = resolveAvailableModel(model, availability);
return resolved ? [resolved] : [];
}),
),
),
]),
) as PreferredTierModels;
const selectPreferredTierModels = (
preferredByTier: PreferredTierModels,
usableNames: ReadonlySet<string>,
): [string, string, string, string] | null => {
const preferred = TIER_NAMES.map((tier) => preferredByTier[tier].find((name) => usableNames.has(name)));
const candidates = preferred.flatMap((model, tier) => (model ? [{ model, tier }] : []));
if (candidates.length === 0) return null;
const nearest = (tier: number): string =>
[...candidates].sort(
(left, right) => Math.abs(left.tier - tier) - Math.abs(right.tier - tier) || left.tier - right.tier,
)[0].model;
return preferred.map((model, tier) => model ?? nearest(tier)) as [string, string, string, string];
};
export const buildAutomaticRouterConfig = (
models: ModelGroup[],
deployments: AutoRouterDeployment[],
preferredByTier: PreferredTierModels,
): ComplexityRouterConfigValue | null => {
const autoRouterNames: ReadonlySet<string> = new Set(
deployments
.filter(isAutoRouterDeployment)
.flatMap((deployment) => (deployment.model_name ? [deployment.model_name] : [])),
);
const names = Array.from(
new Set(
models
.filter((model) => model.mode === undefined || model.mode === "chat")
.map((model) => model.model_group)
.filter((name) => name && !name.startsWith("auto_router/") && !autoRouterNames.has(name)),
),
);
if (names.length === 0) return null;
const usableNames: ReadonlySet<string> = new Set(names);
const selected = selectPreferredTierModels(preferredByTier, usableNames);
if (selected === null) return null;
const supportedReasoningEfforts = models.find(
(model) => model.model_group === selected[3],
)?.supported_reasoning_efforts;
const reasoningEffort = REASONING_EFFORT_STRENGTH.find((effort) => supportedReasoningEfforts?.includes(effort));
return {
tiers: {
SIMPLE: [selected[0]],
MEDIUM: [selected[1]],
COMPLEX: [selected[2]],
REASONING: [selected[3]],
},
classifier_type: "heuristic_v2",
...(reasoningEffort && {
tier_model_params: { REASONING: { [selected[3]]: { reasoning_effort: reasoningEffort } } },
}),
};
};

View file

@ -152,7 +152,7 @@ export const deploymentRefsFromModelInfo = (
return row.model_name && underlyingModels.length > 0 ? [{ modelGroup: row.model_name, underlyingModels }] : [];
});
const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => {
export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => {
const { modelGroups, underlyingIndex } = availability;
if (modelGroups.has(requiredModel)) return requiredModel;
const normalized = normalizeModelName(requiredModel);