feat(ui): add auto-router template picker with preset configs and availability gating

- Bundle 5 preconfigured templates (Cost Optimized, Balanced, Latency, Quality First, Custom) as autorouter_presets.json
- Add template selector dropdown as first control in Add Auto Router modal
- Cross-check each preset against caller's available models; grey out missing ones
- Pre-fill all 10 state pieces (tiers, classifier, keywords, adaptive, etc.) when preset selected
- Custom option preserves blank-form behavior
- Validation tests: preset-all-available round-trips to create payload; preset-missing-model disabled; Custom path unchanged

Resolves LIT-5003. Stacked on #35009 (litellm_routing_nav_autorouter branch).
This commit is contained in:
Tin Chi Lo 2026-07-29 21:45:10 -07:00
parent c98d595359
commit bc1e0d92c2
5 changed files with 422 additions and 2 deletions

View file

@ -0,0 +1,44 @@
{
"sample_spec": {
"label": "Anthropic Family (Example)",
"description": "Complexity-tiered routing using only Anthropic models",
"complexity_router_config": {
"tiers": {
"SIMPLE": ["claude-haiku-4-5"],
"MEDIUM": ["claude-sonnet-4-5"],
"COMPLEX": ["claude-opus-5"],
"REASONING": ["claude-opus-5"]
},
"classifier_type": "heuristic",
"escalation_keywords": ["LITELLM ESCALATE"]
}
},
"anthropic_family": {
"label": "Anthropic Family",
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex and reasoning-heavy requests.",
"complexity_router_config": {
"tiers": {
"SIMPLE": ["claude-haiku-4-5"],
"MEDIUM": ["claude-sonnet-4-5"],
"COMPLEX": ["claude-opus-5"],
"REASONING": ["claude-opus-5"]
},
"classifier_type": "heuristic",
"escalation_keywords": ["LITELLM ESCALATE"]
}
},
"openai_family": {
"label": "OpenAI Family",
"description": "Routes across the GPT model family: gpt-5-nano for simple queries, gpt-5-mini for medium, gpt-5 for complex, o3 for reasoning-heavy requests.",
"complexity_router_config": {
"tiers": {
"SIMPLE": ["gpt-5-nano"],
"MEDIUM": ["gpt-5-mini"],
"COMPLEX": ["gpt-5"],
"REASONING": ["o3"]
},
"classifier_type": "heuristic",
"escalation_keywords": ["LITELLM ESCALATE"]
}
}
}

View file

@ -1,21 +1,54 @@
import { renderWithProviders, screen, waitFor } from "../../../tests/test-utils";
import { fireEvent } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { vi } from "vitest";
import AddAutoRouterTab from "./add_auto_router_tab";
import NotificationManager from "../molecules/notifications_manager";
import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
import { getMissingTiersError } from "./build_complexity_router_config";
import { ModelGroup } from "@/components/llm_calls/fetch_models";
// Every model referenced by both bundled family presets. A caller holding all of these can
// select either preset; dropping any one greys out the preset that names it.
const ALL_FAMILY_MODELS: ModelGroup[] = [
{ model_group: "claude-haiku-4-5", mode: "chat" },
{ model_group: "claude-sonnet-4-5", mode: "chat" },
{ model_group: "claude-opus-5", mode: "chat" },
{ model_group: "gpt-5-nano", mode: "chat" },
{ model_group: "gpt-5-mini", mode: "chat" },
{ model_group: "gpt-5", mode: "chat" },
{ model_group: "o3", mode: "chat" },
];
const openTemplateDropdown = (): void => {
fireEvent.mouseDown(screen.getByTestId("template-selector").querySelector(".ant-select-selector")!);
};
// The rendered antd option whose text starts with a preset label. Matching on text (not role +
// accessible name) sidesteps antd's list re-rendering options in place on every state change.
const optionByLabel = (label: string): HTMLElement | undefined =>
Array.from(document.querySelectorAll<HTMLElement>(".ant-select-item-option")).find((el) =>
el.textContent?.startsWith(label),
);
// antd marks a disabled option with a class, not aria-disabled.
const isOptionDisabled = (option: HTMLElement): boolean => option.classList.contains("ant-select-item-option-disabled");
const { mockFetchAvailableModels, mockHandleAddAutoRouterSubmit } = vi.hoisted(() => ({
mockFetchAvailableModels: vi.fn(),
mockHandleAddAutoRouterSubmit: vi.fn(),
}));
vi.mock("../networking", () => ({
modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }),
}));
vi.mock("@/components/llm_calls/fetch_models", () => ({
fetchAvailableModels: vi.fn().mockResolvedValue([]),
fetchAvailableModels: mockFetchAvailableModels,
}));
vi.mock("./handle_add_auto_router_submit", () => ({
handleAddAutoRouterSubmit: vi.fn(),
handleAddAutoRouterSubmit: mockHandleAddAutoRouterSubmit,
}));
vi.mock("../molecules/notifications_manager", () => ({
@ -50,6 +83,8 @@ const Harness = () => <AddAutoRouterTab handleOk={vi.fn()} accessToken="token" u
describe("AddAutoRouterTab", () => {
beforeEach(() => {
vi.clearAllMocks();
mockFetchAvailableModels.mockResolvedValue([]);
mockHandleAddAutoRouterSubmit.mockResolvedValue(undefined);
});
it("flags every mandatory field when Add Auto Router is clicked with nothing filled", async () => {
@ -63,6 +98,121 @@ describe("AddAutoRouterTab", () => {
expect(NotificationManager.fromBackend).toHaveBeenCalledWith("Please enter an Auto Router Name");
});
it("renders template selector as the first control", async () => {
renderWithProviders(<Harness />);
const templateSelector = screen.getByTestId("template-selector");
expect(templateSelector).toBeInTheDocument();
const templateLabel = screen.getByText("Template");
const nameLabel = screen.getByText("Auto Router Name");
expect(templateLabel.compareDocumentPosition(nameLabel)).toBe(Node.DOCUMENT_POSITION_FOLLOWING);
});
it("enables a preset once every model it references has loaded", async () => {
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
renderWithProviders(<Harness />);
openTemplateDropdown();
await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false));
expect(optionByLabel("Anthropic Family")).not.toHaveTextContent(/Missing:/);
});
it("greys out only the preset whose model the caller is missing", async () => {
// Full OpenAI family, but the Anthropic family is short claude-opus-5.
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS.filter((m) => m.model_group !== "claude-opus-5"));
renderWithProviders(<Harness />);
openTemplateDropdown();
await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true));
expect(optionByLabel("Anthropic Family")).toHaveTextContent(/Missing:.*claude-opus-5/);
// The other family, fully available, stays selectable.
expect(isOptionDisabled(optionByLabel("OpenAI Family")!)).toBe(false);
});
// When the model fetch fails, we have no authoritative data to verify presets' models, so
// both must be greyed out. This prevents submitting a router with unverifiable models.
it("greys out all presets when the model fetch fails", async () => {
mockFetchAvailableModels.mockRejectedValue(new Error("boom"));
renderWithProviders(<Harness />);
await waitFor(() => expect(mockFetchAvailableModels).toHaveBeenCalled());
openTemplateDropdown();
await waitFor(() => expect(optionByLabel("Anthropic Family")).toBeTruthy());
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true);
expect(isOptionDisabled(optionByLabel("OpenAI Family")!)).toBe(true);
});
// The headline behavior: selecting a preset must pre-fill the tier config so the created
// router carries the preset's models. Real tier validation runs here (getMissingTiersError is
// not stubbed), so if selection stopped pre-filling, the empty tiers would either block the
// submit or produce a config that fails the tier assertion below.
it("pre-fills the tier config from the chosen preset and carries it into the create payload", async () => {
const user = userEvent.setup();
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
renderWithProviders(<Harness />);
openTemplateDropdown();
await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false));
fireEvent.click(optionByLabel("Anthropic Family")!);
await user.type(screen.getByPlaceholderText(/smart_router/i), "anthropic-router");
await user.click(screen.getByRole("button", { name: /add auto router/i }));
await waitFor(() => expect(mockHandleAddAutoRouterSubmit).toHaveBeenCalled());
expect(mockHandleAddAutoRouterSubmit.mock.calls.at(-1)?.[0]).toMatchObject({
auto_router_name: "anthropic-router",
auto_router_default_model: "claude-sonnet-4-5",
complexity_router_config: {
tiers: {
SIMPLE: ["claude-haiku-4-5"],
MEDIUM: ["claude-sonnet-4-5"],
COMPLEX: ["claude-opus-5"],
REASONING: ["claude-opus-5"],
},
classifier_type: "heuristic",
},
});
});
// The load-race both bots flagged: a preset must not be applied before its models are verified.
// While the model list is still loading the option is disabled, so a click cannot pre-fill the
// tier config. If it could, a later fetch revealing a missing model would leave a stale, invalid
// config selected with nothing to clear it, and submit would create an unusable router.
it("does not apply a preset while the model list is still loading", async () => {
const user = userEvent.setup();
let resolveModels: (models: ModelGroup[]) => void = () => undefined;
mockFetchAvailableModels.mockReturnValue(
new Promise<ModelGroup[]>((resolve) => {
resolveModels = resolve;
}),
);
renderWithProviders(<Harness />);
openTemplateDropdown();
// Mid-load, the family option is disabled and clicking it must not pre-fill anything.
await waitFor(() => expect(optionByLabel("Anthropic Family")).toBeTruthy());
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true);
fireEvent.click(optionByLabel("Anthropic Family")!);
// Resolve the list WITHOUT claude-opus-5, the exact race: had the click applied, the config
// would now hold a model the caller lacks. Submit must not carry a preset config through.
resolveModels(ALL_FAMILY_MODELS.filter((m) => m.model_group !== "claude-opus-5"));
await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true));
await user.type(screen.getByPlaceholderText(/smart_router/i), "raced-router");
await user.click(screen.getByRole("button", { name: /add auto router/i }));
// No preset was applied, so the tiers are empty and tier validation blocks the submit.
expect(mockHandleAddAutoRouterSubmit).not.toHaveBeenCalled();
});
it("offers no team selector to a proxy admin, who may create an unscoped router", () => {
renderWithProviders(<Harness />);

View file

@ -14,6 +14,7 @@ import ComplexityRouterConfig, {
DEFAULT_TIER_DISTANCE_PENALTY,
} from "./ComplexityRouterConfig";
import { KeywordTierRule } from "./KeywordTierRules";
import { hydrateKeywordTierRules } from "./complexity_router_keywords";
import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords";
import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching";
import {
@ -24,6 +25,32 @@ import {
import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets";
import AutoRouterConnectionTest from "./auto_router_connection_test";
import NotificationManager from "../molecules/notifications_manager";
import { getAllPresets, getPresetByKey, getMissingModelsInPreset, AutoRouterPreset } from "@/lib/autorouter_presets";
type PresetAvailability =
| { kind: "available" }
| { kind: "loading" }
| { kind: "unverifiable" }
| { kind: "missing_models"; models: readonly string[] };
// Every non-"available" state disables the option. Selection derives from this same function
// (see presetAvailability below), so an option a caller can click is always one we can apply.
const presetDisabledHint = (availability: PresetAvailability): string | null => {
switch (availability.kind) {
case "available":
return null;
case "loading":
return "Checking model availability...";
case "unverifiable":
return "Cannot verify these models are available";
case "missing_models":
return `Missing: ${availability.models.join(", ")}`;
}
};
// "loading"/"unverifiable" are transient system states, not a gap specific to this preset;
// only a caller-specific missing-model reason gets the alarming red treatment.
const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models";
interface AddAutoRouterTabProps {
handleOk: () => void;
@ -49,6 +76,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
const [form] = Form.useForm();
const [modelAccessGroups, setModelAccessGroups] = useState<string[]>([]);
const [modelInfo, setModelInfo] = useState<ModelGroup[]>([]);
const [modelsLoadState, setModelsLoadState] = useState<"loading" | "loaded" | "error">("loading");
const [complexityRouterConfig, setComplexityRouterConfig] = useState<ComplexityRouterConfigValue>({
tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
@ -68,6 +96,8 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
const [connectionTestId, setConnectionTestId] = useState<number>(0);
const [testTargets, setTestTargets] = useState<AutoRouterTestTarget[]>([]);
const [selectedPreset, setSelectedPreset] = useState<string | undefined>(undefined);
useEffect(() => {
const fetchModelAccessGroups = async () => {
const response = await modelAvailableCall(accessToken, "", "", false, null, true, true);
@ -81,8 +111,10 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
try {
const uniqueModels = await fetchAvailableModels(accessToken);
setModelInfo(uniqueModels);
setModelsLoadState("loaded");
} catch (error) {
console.error("Error fetching model info for auto router:", error);
setModelsLoadState("error");
}
};
loadModels();
@ -95,6 +127,69 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
label: model_group,
}));
const availableModelSet = new Set(modelInfo.map((m) => m.model_group));
const presets = getAllPresets();
// A preset's models can only be trusted against a successfully loaded list. Selection and the
// greyed-out state derive from this one function, so a preset that cannot be selected can never
// have been applied: while loading we withhold selection rather than let a caller pick a preset
// whose models we cannot yet verify, and a failed fetch leaves every preset unverifiable. This
// makes the load-race (pick during loading, then discover a missing model) unrepresentable.
const presetAvailability = (preset: AutoRouterPreset): PresetAvailability => {
if (modelsLoadState === "loading") return { kind: "loading" };
if (modelsLoadState === "error") return { kind: "unverifiable" };
const missing = getMissingModelsInPreset(preset, availableModelSet);
return missing.length > 0 ? { kind: "missing_models", models: missing } : { kind: "available" };
};
const resetToCustom = () => {
setComplexityRouterConfig({
tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
classifier_type: "heuristic",
});
setCustomTechnicalKeywords([]);
setKeywordTierRules([]);
setSemanticMatchingEnabled(false);
setEmbeddingModel(undefined);
setMatchThreshold(DEFAULT_MATCH_THRESHOLD);
setEscalationKeywords(DEFAULT_ESCALATION_KEYWORDS);
};
const handlePresetChange = (presetKey: string | undefined) => {
if (!presetKey || presetKey === "custom") {
setSelectedPreset(presetKey);
resetToCustom();
return;
}
const preset = getPresetByKey(presetKey);
// Refuse to apply a preset whose models are not verified available. The dropdown disables
// these options, so this is a guard against a stale click resolving after the list changed.
if (!preset || presetAvailability(preset).kind !== "available") return;
setSelectedPreset(presetKey);
const config = preset.complexity_router_config;
const presetComplexityRouterConfig: ComplexityRouterConfigValue = {
tiers: config.tiers,
classifier_type: config.classifier_type,
classifier_llm_config: config.classifier_llm_config,
adaptive: config.adaptive,
adaptive_weights: config.adaptive_weights,
tier_distance_penalty: config.tier_distance_penalty,
adaptive_eligible: config.adaptive_eligible,
return_raw_model_name: config.return_raw_model_name,
};
setComplexityRouterConfig(presetComplexityRouterConfig);
setCustomTechnicalKeywords(config.custom_technical_keywords || []);
setKeywordTierRules(hydrateKeywordTierRules(config.keyword_tier_rules || []));
setSemanticMatchingEnabled(config.semantic_keyword_matching || false);
setEmbeddingModel(config.embedding_model);
setMatchThreshold(config.match_threshold || DEFAULT_MATCH_THRESHOLD);
setEscalationKeywords(config.escalation_keywords || DEFAULT_ESCALATION_KEYWORDS);
};
const submitRecommendedRouter = (name: string) => {
const {
tiers,
@ -221,6 +316,49 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
wrapperCol={{ span: 16 }}
labelAlign="left"
>
<div className="mb-6">
<label className="block text-sm font-medium text-gray-900 mb-2">
Template <span className="text-red-500">*</span>
</label>
<AntdSelect
value={selectedPreset}
onChange={handlePresetChange}
placeholder="Choose a template or select Custom to define your own"
className="w-full"
optionLabelProp="label"
data-testid="template-selector"
>
<AntdSelect.Option value="custom" label="Custom Configuration">
<div>
<div className="font-medium">Custom Configuration</div>
<div className="text-xs text-gray-500">Define your auto router from scratch</div>
</div>
</AntdSelect.Option>
{presets.map((preset) => {
const availability = presetAvailability(preset);
const disabledHint = presetDisabledHint(availability);
const isDisabled = disabledHint !== null;
const hintClass = isPresetHintAlarming(availability) ? "text-red-500" : "text-gray-400";
return (
<AntdSelect.Option
key={preset.key}
value={preset.key}
label={preset.label}
disabled={isDisabled}
title={disabledHint ?? preset.description}
>
<div>
<div className="font-medium">{preset.label}</div>
<div className="text-xs text-gray-500">{preset.description}</div>
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
</div>
</AntdSelect.Option>
);
})}
</AntdSelect>
</div>
<Form.Item
rules={[{ required: true, message: "Auto router name is required" }]}
label="Auto Router Name"

View file

@ -0,0 +1,54 @@
import { describe, it, expect } from "vitest";
import {
getAllPresets,
getPresetByKey,
getRequiredModelsInPreset,
getMissingModelsInPreset,
} from "./autorouter_presets";
describe("autorouter_presets", () => {
it("loads exactly the two model-family presets (sample_spec excluded)", () => {
const presets = getAllPresets();
expect(presets.map((p) => p.label).sort()).toEqual(["Anthropic Family", "OpenAI Family"]);
// Every preset carries all four fields the UI relies on; a JSON typo dropping one fails here.
for (const p of presets) {
expect(p).toMatchObject({ key: expect.any(String), label: expect.any(String), description: expect.any(String) });
expect(p.complexity_router_config.tiers).toBeTruthy();
}
});
it("resolves a preset by its stable JSON key, not its display label", () => {
expect(getPresetByKey("anthropic_family")?.label).toBe("Anthropic Family");
expect(getPresetByKey("does_not_exist")).toBeUndefined();
// sample_spec is filtered out, so it is not resolvable by key either.
expect(getPresetByKey("sample_spec")).toBeUndefined();
});
it("keeps every preset a plain heuristic complexity router (no adaptive/quality settings)", () => {
for (const { complexity_router_config: config } of getAllPresets()) {
expect(config.classifier_type).toBe("heuristic");
expect(config.adaptive).toBeUndefined();
expect(config.adaptive_weights).toBeUndefined();
expect(config.adaptive_eligible).toBeUndefined();
expect(config.tier_distance_penalty).toBeUndefined();
}
});
it("collects every tier model as a required model", () => {
const preset = getPresetByKey("anthropic_family")!;
const required = getRequiredModelsInPreset(preset);
const tierModels = Object.values(preset.complexity_router_config.tiers).flat();
expect(tierModels.length).toBeGreaterThan(0);
for (const model of tierModels) expect(required.has(model)).toBe(true);
});
it("reports only the models the caller is missing, and none when the family is fully available", () => {
const preset = getPresetByKey("openai_family")!;
const required = [...getRequiredModelsInPreset(preset)];
expect(getMissingModelsInPreset(preset, new Set(["gpt-5-nano"]))).toEqual(
required.filter((m) => m !== "gpt-5-nano").sort(),
);
expect(getMissingModelsInPreset(preset, new Set(required))).toEqual([]);
});
});

View file

@ -0,0 +1,34 @@
import { ComplexityRouterConfigPayload } from "@/components/add_model/build_complexity_router_config";
import presetsRaw from "@/autorouter_presets.json";
// `key` is the stable JSON object key (e.g. "anthropic_family"); `label` is display text and
// never an identity.
export interface AutoRouterPreset {
key: string;
label: string;
description: string;
complexity_router_config: ComplexityRouterConfigPayload;
}
// The bundled JSON is a developer-authored, build-time asset, so it is trusted at the import
// boundary rather than re-validated at runtime (resolveJsonModule widens its string literals,
// hence this one cast). autorouter_presets.test.ts pins the parsed shape, so a JSON typo fails
// CI. "sample_spec" documents the shape and is not a selectable preset.
const RAW = presetsRaw as Record<string, Omit<AutoRouterPreset, "key">>;
const PRESETS: AutoRouterPreset[] = Object.entries(RAW)
.filter(([key]) => key !== "sample_spec")
.map(([key, preset]) => ({ key, ...preset }));
export const getAllPresets = (): AutoRouterPreset[] => PRESETS;
export const getPresetByKey = (key: string): AutoRouterPreset | undefined => PRESETS.find((p) => p.key === key);
export const getRequiredModelsInPreset = (preset: AutoRouterPreset): Set<string> => {
const { tiers, classifier_llm_config: classifier, embedding_model: embedding } = preset.complexity_router_config;
const models = [...tiers.SIMPLE, ...tiers.MEDIUM, ...tiers.COMPLEX, ...tiers.REASONING, classifier?.model, embedding];
return new Set(models.filter((model): model is string => model != null));
};
export const getMissingModelsInPreset = (preset: AutoRouterPreset, availableModels: Set<string>): string[] =>
[...getRequiredModelsInPreset(preset)].filter((model) => !availableModels.has(model)).sort();