diff --git a/ui/litellm-dashboard/src/autorouter_presets.json b/ui/litellm-dashboard/src/autorouter_presets.json new file mode 100644 index 00000000000..b1c0235eb13 --- /dev/null +++ b/ui/litellm-dashboard/src/autorouter_presets.json @@ -0,0 +1,44 @@ +{ + "sample_spec": { + "label": "Anthropic Family (Example)", + "description": "Complexity-tiered routing using only Anthropic models", + "complexity_router_config": { + "tiers": { + "SIMPLE": ["claude-haiku-4-5"], + "MEDIUM": ["claude-sonnet-4-5"], + "COMPLEX": ["claude-opus-5"], + "REASONING": ["claude-opus-5"] + }, + "classifier_type": "heuristic", + "escalation_keywords": ["LITELLM ESCALATE"] + } + }, + "anthropic_family": { + "label": "Anthropic Family", + "description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex and reasoning-heavy requests.", + "complexity_router_config": { + "tiers": { + "SIMPLE": ["claude-haiku-4-5"], + "MEDIUM": ["claude-sonnet-4-5"], + "COMPLEX": ["claude-opus-5"], + "REASONING": ["claude-opus-5"] + }, + "classifier_type": "heuristic", + "escalation_keywords": ["LITELLM ESCALATE"] + } + }, + "openai_family": { + "label": "OpenAI Family", + "description": "Routes across the GPT model family: gpt-5-nano for simple queries, gpt-5-mini for medium, gpt-5 for complex, o3 for reasoning-heavy requests.", + "complexity_router_config": { + "tiers": { + "SIMPLE": ["gpt-5-nano"], + "MEDIUM": ["gpt-5-mini"], + "COMPLEX": ["gpt-5"], + "REASONING": ["o3"] + }, + "classifier_type": "heuristic", + "escalation_keywords": ["LITELLM ESCALATE"] + } + } +} diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index f7cc9a1deae..c05f533297f 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -1,21 +1,54 @@ import { renderWithProviders, screen, waitFor } from "../../../tests/test-utils"; +import { fireEvent } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { vi } from "vitest"; import AddAutoRouterTab from "./add_auto_router_tab"; import NotificationManager from "../molecules/notifications_manager"; import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; import { getMissingTiersError } from "./build_complexity_router_config"; +import { ModelGroup } from "@/components/llm_calls/fetch_models"; + +// Every model referenced by both bundled family presets. A caller holding all of these can +// select either preset; dropping any one greys out the preset that names it. +const ALL_FAMILY_MODELS: ModelGroup[] = [ + { model_group: "claude-haiku-4-5", mode: "chat" }, + { model_group: "claude-sonnet-4-5", mode: "chat" }, + { model_group: "claude-opus-5", mode: "chat" }, + { model_group: "gpt-5-nano", mode: "chat" }, + { model_group: "gpt-5-mini", mode: "chat" }, + { model_group: "gpt-5", mode: "chat" }, + { model_group: "o3", mode: "chat" }, +]; + +const openTemplateDropdown = (): void => { + fireEvent.mouseDown(screen.getByTestId("template-selector").querySelector(".ant-select-selector")!); +}; + +// The rendered antd option whose text starts with a preset label. Matching on text (not role + +// accessible name) sidesteps antd's list re-rendering options in place on every state change. +const optionByLabel = (label: string): HTMLElement | undefined => + Array.from(document.querySelectorAll(".ant-select-item-option")).find((el) => + el.textContent?.startsWith(label), + ); + +// antd marks a disabled option with a class, not aria-disabled. +const isOptionDisabled = (option: HTMLElement): boolean => option.classList.contains("ant-select-item-option-disabled"); + +const { mockFetchAvailableModels, mockHandleAddAutoRouterSubmit } = vi.hoisted(() => ({ + mockFetchAvailableModels: vi.fn(), + mockHandleAddAutoRouterSubmit: vi.fn(), +})); vi.mock("../networking", () => ({ modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }), })); vi.mock("@/components/llm_calls/fetch_models", () => ({ - fetchAvailableModels: vi.fn().mockResolvedValue([]), + fetchAvailableModels: mockFetchAvailableModels, })); vi.mock("./handle_add_auto_router_submit", () => ({ - handleAddAutoRouterSubmit: vi.fn(), + handleAddAutoRouterSubmit: mockHandleAddAutoRouterSubmit, })); vi.mock("../molecules/notifications_manager", () => ({ @@ -50,6 +83,8 @@ const Harness = () => { beforeEach(() => { vi.clearAllMocks(); + mockFetchAvailableModels.mockResolvedValue([]); + mockHandleAddAutoRouterSubmit.mockResolvedValue(undefined); }); it("flags every mandatory field when Add Auto Router is clicked with nothing filled", async () => { @@ -63,6 +98,121 @@ describe("AddAutoRouterTab", () => { expect(NotificationManager.fromBackend).toHaveBeenCalledWith("Please enter an Auto Router Name"); }); + it("renders template selector as the first control", async () => { + renderWithProviders(); + + const templateSelector = screen.getByTestId("template-selector"); + expect(templateSelector).toBeInTheDocument(); + + const templateLabel = screen.getByText("Template"); + const nameLabel = screen.getByText("Auto Router Name"); + + expect(templateLabel.compareDocumentPosition(nameLabel)).toBe(Node.DOCUMENT_POSITION_FOLLOWING); + }); + + it("enables a preset once every model it references has loaded", async () => { + mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); + + renderWithProviders(); + openTemplateDropdown(); + + await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false)); + expect(optionByLabel("Anthropic Family")).not.toHaveTextContent(/Missing:/); + }); + + it("greys out only the preset whose model the caller is missing", async () => { + // Full OpenAI family, but the Anthropic family is short claude-opus-5. + mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS.filter((m) => m.model_group !== "claude-opus-5")); + + renderWithProviders(); + openTemplateDropdown(); + + await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true)); + expect(optionByLabel("Anthropic Family")).toHaveTextContent(/Missing:.*claude-opus-5/); + // The other family, fully available, stays selectable. + expect(isOptionDisabled(optionByLabel("OpenAI Family")!)).toBe(false); + }); + + // When the model fetch fails, we have no authoritative data to verify presets' models, so + // both must be greyed out. This prevents submitting a router with unverifiable models. + it("greys out all presets when the model fetch fails", async () => { + mockFetchAvailableModels.mockRejectedValue(new Error("boom")); + + renderWithProviders(); + await waitFor(() => expect(mockFetchAvailableModels).toHaveBeenCalled()); + openTemplateDropdown(); + + await waitFor(() => expect(optionByLabel("Anthropic Family")).toBeTruthy()); + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true); + expect(isOptionDisabled(optionByLabel("OpenAI Family")!)).toBe(true); + }); + + // The headline behavior: selecting a preset must pre-fill the tier config so the created + // router carries the preset's models. Real tier validation runs here (getMissingTiersError is + // not stubbed), so if selection stopped pre-filling, the empty tiers would either block the + // submit or produce a config that fails the tier assertion below. + it("pre-fills the tier config from the chosen preset and carries it into the create payload", async () => { + const user = userEvent.setup(); + mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); + + renderWithProviders(); + openTemplateDropdown(); + + await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false)); + fireEvent.click(optionByLabel("Anthropic Family")!); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "anthropic-router"); + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(mockHandleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(mockHandleAddAutoRouterSubmit.mock.calls.at(-1)?.[0]).toMatchObject({ + auto_router_name: "anthropic-router", + auto_router_default_model: "claude-sonnet-4-5", + complexity_router_config: { + tiers: { + SIMPLE: ["claude-haiku-4-5"], + MEDIUM: ["claude-sonnet-4-5"], + COMPLEX: ["claude-opus-5"], + REASONING: ["claude-opus-5"], + }, + classifier_type: "heuristic", + }, + }); + }); + + // The load-race both bots flagged: a preset must not be applied before its models are verified. + // While the model list is still loading the option is disabled, so a click cannot pre-fill the + // tier config. If it could, a later fetch revealing a missing model would leave a stale, invalid + // config selected with nothing to clear it, and submit would create an unusable router. + it("does not apply a preset while the model list is still loading", async () => { + const user = userEvent.setup(); + let resolveModels: (models: ModelGroup[]) => void = () => undefined; + mockFetchAvailableModels.mockReturnValue( + new Promise((resolve) => { + resolveModels = resolve; + }), + ); + + renderWithProviders(); + openTemplateDropdown(); + + // Mid-load, the family option is disabled and clicking it must not pre-fill anything. + await waitFor(() => expect(optionByLabel("Anthropic Family")).toBeTruthy()); + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true); + fireEvent.click(optionByLabel("Anthropic Family")!); + + // Resolve the list WITHOUT claude-opus-5, the exact race: had the click applied, the config + // would now hold a model the caller lacks. Submit must not carry a preset config through. + resolveModels(ALL_FAMILY_MODELS.filter((m) => m.model_group !== "claude-opus-5")); + await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true)); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "raced-router"); + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + // No preset was applied, so the tiers are empty and tier validation blocks the submit. + expect(mockHandleAddAutoRouterSubmit).not.toHaveBeenCalled(); + }); + it("offers no team selector to a proxy admin, who may create an unscoped router", () => { renderWithProviders(); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index ea75bd8e283..5c31b229fc9 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -14,6 +14,7 @@ import ComplexityRouterConfig, { DEFAULT_TIER_DISTANCE_PENALTY, } from "./ComplexityRouterConfig"; import { KeywordTierRule } from "./KeywordTierRules"; +import { hydrateKeywordTierRules } from "./complexity_router_keywords"; import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords"; import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching"; import { @@ -24,6 +25,32 @@ import { import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets"; import AutoRouterConnectionTest from "./auto_router_connection_test"; import NotificationManager from "../molecules/notifications_manager"; +import { getAllPresets, getPresetByKey, getMissingModelsInPreset, AutoRouterPreset } from "@/lib/autorouter_presets"; + +type PresetAvailability = + | { kind: "available" } + | { kind: "loading" } + | { kind: "unverifiable" } + | { kind: "missing_models"; models: readonly string[] }; + +// Every non-"available" state disables the option. Selection derives from this same function +// (see presetAvailability below), so an option a caller can click is always one we can apply. +const presetDisabledHint = (availability: PresetAvailability): string | null => { + switch (availability.kind) { + case "available": + return null; + case "loading": + return "Checking model availability..."; + case "unverifiable": + return "Cannot verify these models are available"; + case "missing_models": + return `Missing: ${availability.models.join(", ")}`; + } +}; + +// "loading"/"unverifiable" are transient system states, not a gap specific to this preset; +// only a caller-specific missing-model reason gets the alarming red treatment. +const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models"; interface AddAutoRouterTabProps { handleOk: () => void; @@ -49,6 +76,7 @@ const AddAutoRouterTab: React.FC = ({ const [form] = Form.useForm(); const [modelAccessGroups, setModelAccessGroups] = useState([]); const [modelInfo, setModelInfo] = useState([]); + const [modelsLoadState, setModelsLoadState] = useState<"loading" | "loaded" | "error">("loading"); const [complexityRouterConfig, setComplexityRouterConfig] = useState({ tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, @@ -68,6 +96,8 @@ const AddAutoRouterTab: React.FC = ({ const [connectionTestId, setConnectionTestId] = useState(0); const [testTargets, setTestTargets] = useState([]); + const [selectedPreset, setSelectedPreset] = useState(undefined); + useEffect(() => { const fetchModelAccessGroups = async () => { const response = await modelAvailableCall(accessToken, "", "", false, null, true, true); @@ -81,8 +111,10 @@ const AddAutoRouterTab: React.FC = ({ try { const uniqueModels = await fetchAvailableModels(accessToken); setModelInfo(uniqueModels); + setModelsLoadState("loaded"); } catch (error) { console.error("Error fetching model info for auto router:", error); + setModelsLoadState("error"); } }; loadModels(); @@ -95,6 +127,69 @@ const AddAutoRouterTab: React.FC = ({ label: model_group, })); + const availableModelSet = new Set(modelInfo.map((m) => m.model_group)); + const presets = getAllPresets(); + + // A preset's models can only be trusted against a successfully loaded list. Selection and the + // greyed-out state derive from this one function, so a preset that cannot be selected can never + // have been applied: while loading we withhold selection rather than let a caller pick a preset + // whose models we cannot yet verify, and a failed fetch leaves every preset unverifiable. This + // makes the load-race (pick during loading, then discover a missing model) unrepresentable. + const presetAvailability = (preset: AutoRouterPreset): PresetAvailability => { + if (modelsLoadState === "loading") return { kind: "loading" }; + if (modelsLoadState === "error") return { kind: "unverifiable" }; + const missing = getMissingModelsInPreset(preset, availableModelSet); + return missing.length > 0 ? { kind: "missing_models", models: missing } : { kind: "available" }; + }; + + const resetToCustom = () => { + setComplexityRouterConfig({ + tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "heuristic", + }); + setCustomTechnicalKeywords([]); + setKeywordTierRules([]); + setSemanticMatchingEnabled(false); + setEmbeddingModel(undefined); + setMatchThreshold(DEFAULT_MATCH_THRESHOLD); + setEscalationKeywords(DEFAULT_ESCALATION_KEYWORDS); + }; + + const handlePresetChange = (presetKey: string | undefined) => { + if (!presetKey || presetKey === "custom") { + setSelectedPreset(presetKey); + resetToCustom(); + return; + } + + const preset = getPresetByKey(presetKey); + // Refuse to apply a preset whose models are not verified available. The dropdown disables + // these options, so this is a guard against a stale click resolving after the list changed. + if (!preset || presetAvailability(preset).kind !== "available") return; + + setSelectedPreset(presetKey); + + const config = preset.complexity_router_config; + const presetComplexityRouterConfig: ComplexityRouterConfigValue = { + tiers: config.tiers, + classifier_type: config.classifier_type, + classifier_llm_config: config.classifier_llm_config, + adaptive: config.adaptive, + adaptive_weights: config.adaptive_weights, + tier_distance_penalty: config.tier_distance_penalty, + adaptive_eligible: config.adaptive_eligible, + return_raw_model_name: config.return_raw_model_name, + }; + setComplexityRouterConfig(presetComplexityRouterConfig); + + setCustomTechnicalKeywords(config.custom_technical_keywords || []); + setKeywordTierRules(hydrateKeywordTierRules(config.keyword_tier_rules || [])); + setSemanticMatchingEnabled(config.semantic_keyword_matching || false); + setEmbeddingModel(config.embedding_model); + setMatchThreshold(config.match_threshold || DEFAULT_MATCH_THRESHOLD); + setEscalationKeywords(config.escalation_keywords || DEFAULT_ESCALATION_KEYWORDS); + }; + const submitRecommendedRouter = (name: string) => { const { tiers, @@ -221,6 +316,49 @@ const AddAutoRouterTab: React.FC = ({ wrapperCol={{ span: 16 }} labelAlign="left" > +
+ + + +
+
Custom Configuration
+
Define your auto router from scratch
+
+ + {presets.map((preset) => { + const availability = presetAvailability(preset); + const disabledHint = presetDisabledHint(availability); + const isDisabled = disabledHint !== null; + const hintClass = isPresetHintAlarming(availability) ? "text-red-500" : "text-gray-400"; + + return ( + +
+
{preset.label}
+
{preset.description}
+ {disabledHint &&
{disabledHint}
} +
+ + ); + })} + +
+ { + it("loads exactly the two model-family presets (sample_spec excluded)", () => { + const presets = getAllPresets(); + expect(presets.map((p) => p.label).sort()).toEqual(["Anthropic Family", "OpenAI Family"]); + // Every preset carries all four fields the UI relies on; a JSON typo dropping one fails here. + for (const p of presets) { + expect(p).toMatchObject({ key: expect.any(String), label: expect.any(String), description: expect.any(String) }); + expect(p.complexity_router_config.tiers).toBeTruthy(); + } + }); + + it("resolves a preset by its stable JSON key, not its display label", () => { + expect(getPresetByKey("anthropic_family")?.label).toBe("Anthropic Family"); + expect(getPresetByKey("does_not_exist")).toBeUndefined(); + // sample_spec is filtered out, so it is not resolvable by key either. + expect(getPresetByKey("sample_spec")).toBeUndefined(); + }); + + it("keeps every preset a plain heuristic complexity router (no adaptive/quality settings)", () => { + for (const { complexity_router_config: config } of getAllPresets()) { + expect(config.classifier_type).toBe("heuristic"); + expect(config.adaptive).toBeUndefined(); + expect(config.adaptive_weights).toBeUndefined(); + expect(config.adaptive_eligible).toBeUndefined(); + expect(config.tier_distance_penalty).toBeUndefined(); + } + }); + + it("collects every tier model as a required model", () => { + const preset = getPresetByKey("anthropic_family")!; + const required = getRequiredModelsInPreset(preset); + const tierModels = Object.values(preset.complexity_router_config.tiers).flat(); + expect(tierModels.length).toBeGreaterThan(0); + for (const model of tierModels) expect(required.has(model)).toBe(true); + }); + + it("reports only the models the caller is missing, and none when the family is fully available", () => { + const preset = getPresetByKey("openai_family")!; + const required = [...getRequiredModelsInPreset(preset)]; + + expect(getMissingModelsInPreset(preset, new Set(["gpt-5-nano"]))).toEqual( + required.filter((m) => m !== "gpt-5-nano").sort(), + ); + expect(getMissingModelsInPreset(preset, new Set(required))).toEqual([]); + }); +}); diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts new file mode 100644 index 00000000000..46aec8685df --- /dev/null +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -0,0 +1,34 @@ +import { ComplexityRouterConfigPayload } from "@/components/add_model/build_complexity_router_config"; +import presetsRaw from "@/autorouter_presets.json"; + +// `key` is the stable JSON object key (e.g. "anthropic_family"); `label` is display text and +// never an identity. +export interface AutoRouterPreset { + key: string; + label: string; + description: string; + complexity_router_config: ComplexityRouterConfigPayload; +} + +// The bundled JSON is a developer-authored, build-time asset, so it is trusted at the import +// boundary rather than re-validated at runtime (resolveJsonModule widens its string literals, +// hence this one cast). autorouter_presets.test.ts pins the parsed shape, so a JSON typo fails +// CI. "sample_spec" documents the shape and is not a selectable preset. +const RAW = presetsRaw as Record>; + +const PRESETS: AutoRouterPreset[] = Object.entries(RAW) + .filter(([key]) => key !== "sample_spec") + .map(([key, preset]) => ({ key, ...preset })); + +export const getAllPresets = (): AutoRouterPreset[] => PRESETS; + +export const getPresetByKey = (key: string): AutoRouterPreset | undefined => PRESETS.find((p) => p.key === key); + +export const getRequiredModelsInPreset = (preset: AutoRouterPreset): Set => { + const { tiers, classifier_llm_config: classifier, embedding_model: embedding } = preset.complexity_router_config; + const models = [...tiers.SIMPLE, ...tiers.MEDIUM, ...tiers.COMPLEX, ...tiers.REASONING, classifier?.model, embedding]; + return new Set(models.filter((model): model is string => model != null)); +}; + +export const getMissingModelsInPreset = (preset: AutoRouterPreset, availableModels: Set): string[] => + [...getRequiredModelsInPreset(preset)].filter((model) => !availableModels.has(model)).sort();