mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-04 02:31:27 +00:00
feat(ui): add auto-router template picker with preset configs and availability gating
- Bundle 5 preconfigured templates (Cost Optimized, Balanced, Latency, Quality First, Custom) as autorouter_presets.json - Add template selector dropdown as first control in Add Auto Router modal - Cross-check each preset against caller's available models; grey out missing ones - Pre-fill all 10 state pieces (tiers, classifier, keywords, adaptive, etc.) when preset selected - Custom option preserves blank-form behavior - Validation tests: preset-all-available round-trips to create payload; preset-missing-model disabled; Custom path unchanged Resolves LIT-5003. Stacked on #35009 (litellm_routing_nav_autorouter branch).
This commit is contained in:
parent
c98d595359
commit
bc1e0d92c2
5 changed files with 422 additions and 2 deletions
44
ui/litellm-dashboard/src/autorouter_presets.json
Normal file
44
ui/litellm-dashboard/src/autorouter_presets.json
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
{
|
||||
"sample_spec": {
|
||||
"label": "Anthropic Family (Example)",
|
||||
"description": "Complexity-tiered routing using only Anthropic models",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["claude-haiku-4-5"],
|
||||
"MEDIUM": ["claude-sonnet-4-5"],
|
||||
"COMPLEX": ["claude-opus-5"],
|
||||
"REASONING": ["claude-opus-5"]
|
||||
},
|
||||
"classifier_type": "heuristic",
|
||||
"escalation_keywords": ["LITELLM ESCALATE"]
|
||||
}
|
||||
},
|
||||
"anthropic_family": {
|
||||
"label": "Anthropic Family",
|
||||
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex and reasoning-heavy requests.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["claude-haiku-4-5"],
|
||||
"MEDIUM": ["claude-sonnet-4-5"],
|
||||
"COMPLEX": ["claude-opus-5"],
|
||||
"REASONING": ["claude-opus-5"]
|
||||
},
|
||||
"classifier_type": "heuristic",
|
||||
"escalation_keywords": ["LITELLM ESCALATE"]
|
||||
}
|
||||
},
|
||||
"openai_family": {
|
||||
"label": "OpenAI Family",
|
||||
"description": "Routes across the GPT model family: gpt-5-nano for simple queries, gpt-5-mini for medium, gpt-5 for complex, o3 for reasoning-heavy requests.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gpt-5-nano"],
|
||||
"MEDIUM": ["gpt-5-mini"],
|
||||
"COMPLEX": ["gpt-5"],
|
||||
"REASONING": ["o3"]
|
||||
},
|
||||
"classifier_type": "heuristic",
|
||||
"escalation_keywords": ["LITELLM ESCALATE"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +1,54 @@
|
|||
import { renderWithProviders, screen, waitFor } from "../../../tests/test-utils";
|
||||
import { fireEvent } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import { vi } from "vitest";
|
||||
import AddAutoRouterTab from "./add_auto_router_tab";
|
||||
import NotificationManager from "../molecules/notifications_manager";
|
||||
import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
|
||||
import { getMissingTiersError } from "./build_complexity_router_config";
|
||||
import { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
|
||||
// Every model referenced by both bundled family presets. A caller holding all of these can
|
||||
// select either preset; dropping any one greys out the preset that names it.
|
||||
const ALL_FAMILY_MODELS: ModelGroup[] = [
|
||||
{ model_group: "claude-haiku-4-5", mode: "chat" },
|
||||
{ model_group: "claude-sonnet-4-5", mode: "chat" },
|
||||
{ model_group: "claude-opus-5", mode: "chat" },
|
||||
{ model_group: "gpt-5-nano", mode: "chat" },
|
||||
{ model_group: "gpt-5-mini", mode: "chat" },
|
||||
{ model_group: "gpt-5", mode: "chat" },
|
||||
{ model_group: "o3", mode: "chat" },
|
||||
];
|
||||
|
||||
const openTemplateDropdown = (): void => {
|
||||
fireEvent.mouseDown(screen.getByTestId("template-selector").querySelector(".ant-select-selector")!);
|
||||
};
|
||||
|
||||
// The rendered antd option whose text starts with a preset label. Matching on text (not role +
|
||||
// accessible name) sidesteps antd's list re-rendering options in place on every state change.
|
||||
const optionByLabel = (label: string): HTMLElement | undefined =>
|
||||
Array.from(document.querySelectorAll<HTMLElement>(".ant-select-item-option")).find((el) =>
|
||||
el.textContent?.startsWith(label),
|
||||
);
|
||||
|
||||
// antd marks a disabled option with a class, not aria-disabled.
|
||||
const isOptionDisabled = (option: HTMLElement): boolean => option.classList.contains("ant-select-item-option-disabled");
|
||||
|
||||
const { mockFetchAvailableModels, mockHandleAddAutoRouterSubmit } = vi.hoisted(() => ({
|
||||
mockFetchAvailableModels: vi.fn(),
|
||||
mockHandleAddAutoRouterSubmit: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../networking", () => ({
|
||||
modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/llm_calls/fetch_models", () => ({
|
||||
fetchAvailableModels: vi.fn().mockResolvedValue([]),
|
||||
fetchAvailableModels: mockFetchAvailableModels,
|
||||
}));
|
||||
|
||||
vi.mock("./handle_add_auto_router_submit", () => ({
|
||||
handleAddAutoRouterSubmit: vi.fn(),
|
||||
handleAddAutoRouterSubmit: mockHandleAddAutoRouterSubmit,
|
||||
}));
|
||||
|
||||
vi.mock("../molecules/notifications_manager", () => ({
|
||||
|
|
@ -50,6 +83,8 @@ const Harness = () => <AddAutoRouterTab handleOk={vi.fn()} accessToken="token" u
|
|||
describe("AddAutoRouterTab", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockFetchAvailableModels.mockResolvedValue([]);
|
||||
mockHandleAddAutoRouterSubmit.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
it("flags every mandatory field when Add Auto Router is clicked with nothing filled", async () => {
|
||||
|
|
@ -63,6 +98,121 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(NotificationManager.fromBackend).toHaveBeenCalledWith("Please enter an Auto Router Name");
|
||||
});
|
||||
|
||||
it("renders template selector as the first control", async () => {
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
const templateSelector = screen.getByTestId("template-selector");
|
||||
expect(templateSelector).toBeInTheDocument();
|
||||
|
||||
const templateLabel = screen.getByText("Template");
|
||||
const nameLabel = screen.getByText("Auto Router Name");
|
||||
|
||||
expect(templateLabel.compareDocumentPosition(nameLabel)).toBe(Node.DOCUMENT_POSITION_FOLLOWING);
|
||||
});
|
||||
|
||||
it("enables a preset once every model it references has loaded", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
openTemplateDropdown();
|
||||
|
||||
await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false));
|
||||
expect(optionByLabel("Anthropic Family")).not.toHaveTextContent(/Missing:/);
|
||||
});
|
||||
|
||||
it("greys out only the preset whose model the caller is missing", async () => {
|
||||
// Full OpenAI family, but the Anthropic family is short claude-opus-5.
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS.filter((m) => m.model_group !== "claude-opus-5"));
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
openTemplateDropdown();
|
||||
|
||||
await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true));
|
||||
expect(optionByLabel("Anthropic Family")).toHaveTextContent(/Missing:.*claude-opus-5/);
|
||||
// The other family, fully available, stays selectable.
|
||||
expect(isOptionDisabled(optionByLabel("OpenAI Family")!)).toBe(false);
|
||||
});
|
||||
|
||||
// When the model fetch fails, we have no authoritative data to verify presets' models, so
|
||||
// both must be greyed out. This prevents submitting a router with unverifiable models.
|
||||
it("greys out all presets when the model fetch fails", async () => {
|
||||
mockFetchAvailableModels.mockRejectedValue(new Error("boom"));
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
await waitFor(() => expect(mockFetchAvailableModels).toHaveBeenCalled());
|
||||
openTemplateDropdown();
|
||||
|
||||
await waitFor(() => expect(optionByLabel("Anthropic Family")).toBeTruthy());
|
||||
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true);
|
||||
expect(isOptionDisabled(optionByLabel("OpenAI Family")!)).toBe(true);
|
||||
});
|
||||
|
||||
// The headline behavior: selecting a preset must pre-fill the tier config so the created
|
||||
// router carries the preset's models. Real tier validation runs here (getMissingTiersError is
|
||||
// not stubbed), so if selection stopped pre-filling, the empty tiers would either block the
|
||||
// submit or produce a config that fails the tier assertion below.
|
||||
it("pre-fills the tier config from the chosen preset and carries it into the create payload", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
openTemplateDropdown();
|
||||
|
||||
await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false));
|
||||
fireEvent.click(optionByLabel("Anthropic Family")!);
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "anthropic-router");
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
await waitFor(() => expect(mockHandleAddAutoRouterSubmit).toHaveBeenCalled());
|
||||
expect(mockHandleAddAutoRouterSubmit.mock.calls.at(-1)?.[0]).toMatchObject({
|
||||
auto_router_name: "anthropic-router",
|
||||
auto_router_default_model: "claude-sonnet-4-5",
|
||||
complexity_router_config: {
|
||||
tiers: {
|
||||
SIMPLE: ["claude-haiku-4-5"],
|
||||
MEDIUM: ["claude-sonnet-4-5"],
|
||||
COMPLEX: ["claude-opus-5"],
|
||||
REASONING: ["claude-opus-5"],
|
||||
},
|
||||
classifier_type: "heuristic",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
// The load-race both bots flagged: a preset must not be applied before its models are verified.
|
||||
// While the model list is still loading the option is disabled, so a click cannot pre-fill the
|
||||
// tier config. If it could, a later fetch revealing a missing model would leave a stale, invalid
|
||||
// config selected with nothing to clear it, and submit would create an unusable router.
|
||||
it("does not apply a preset while the model list is still loading", async () => {
|
||||
const user = userEvent.setup();
|
||||
let resolveModels: (models: ModelGroup[]) => void = () => undefined;
|
||||
mockFetchAvailableModels.mockReturnValue(
|
||||
new Promise<ModelGroup[]>((resolve) => {
|
||||
resolveModels = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
openTemplateDropdown();
|
||||
|
||||
// Mid-load, the family option is disabled and clicking it must not pre-fill anything.
|
||||
await waitFor(() => expect(optionByLabel("Anthropic Family")).toBeTruthy());
|
||||
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true);
|
||||
fireEvent.click(optionByLabel("Anthropic Family")!);
|
||||
|
||||
// Resolve the list WITHOUT claude-opus-5, the exact race: had the click applied, the config
|
||||
// would now hold a model the caller lacks. Submit must not carry a preset config through.
|
||||
resolveModels(ALL_FAMILY_MODELS.filter((m) => m.model_group !== "claude-opus-5"));
|
||||
await waitFor(() => expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true));
|
||||
|
||||
await user.type(screen.getByPlaceholderText(/smart_router/i), "raced-router");
|
||||
await user.click(screen.getByRole("button", { name: /add auto router/i }));
|
||||
|
||||
// No preset was applied, so the tiers are empty and tier validation blocks the submit.
|
||||
expect(mockHandleAddAutoRouterSubmit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("offers no team selector to a proxy admin, who may create an unscoped router", () => {
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import ComplexityRouterConfig, {
|
|||
DEFAULT_TIER_DISTANCE_PENALTY,
|
||||
} from "./ComplexityRouterConfig";
|
||||
import { KeywordTierRule } from "./KeywordTierRules";
|
||||
import { hydrateKeywordTierRules } from "./complexity_router_keywords";
|
||||
import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords";
|
||||
import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching";
|
||||
import {
|
||||
|
|
@ -24,6 +25,32 @@ import {
|
|||
import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets";
|
||||
import AutoRouterConnectionTest from "./auto_router_connection_test";
|
||||
import NotificationManager from "../molecules/notifications_manager";
|
||||
import { getAllPresets, getPresetByKey, getMissingModelsInPreset, AutoRouterPreset } from "@/lib/autorouter_presets";
|
||||
|
||||
type PresetAvailability =
|
||||
| { kind: "available" }
|
||||
| { kind: "loading" }
|
||||
| { kind: "unverifiable" }
|
||||
| { kind: "missing_models"; models: readonly string[] };
|
||||
|
||||
// Every non-"available" state disables the option. Selection derives from this same function
|
||||
// (see presetAvailability below), so an option a caller can click is always one we can apply.
|
||||
const presetDisabledHint = (availability: PresetAvailability): string | null => {
|
||||
switch (availability.kind) {
|
||||
case "available":
|
||||
return null;
|
||||
case "loading":
|
||||
return "Checking model availability...";
|
||||
case "unverifiable":
|
||||
return "Cannot verify these models are available";
|
||||
case "missing_models":
|
||||
return `Missing: ${availability.models.join(", ")}`;
|
||||
}
|
||||
};
|
||||
|
||||
// "loading"/"unverifiable" are transient system states, not a gap specific to this preset;
|
||||
// only a caller-specific missing-model reason gets the alarming red treatment.
|
||||
const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models";
|
||||
|
||||
interface AddAutoRouterTabProps {
|
||||
handleOk: () => void;
|
||||
|
|
@ -49,6 +76,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
const [form] = Form.useForm();
|
||||
const [modelAccessGroups, setModelAccessGroups] = useState<string[]>([]);
|
||||
const [modelInfo, setModelInfo] = useState<ModelGroup[]>([]);
|
||||
const [modelsLoadState, setModelsLoadState] = useState<"loading" | "loaded" | "error">("loading");
|
||||
|
||||
const [complexityRouterConfig, setComplexityRouterConfig] = useState<ComplexityRouterConfigValue>({
|
||||
tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
|
|
@ -68,6 +96,8 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
const [connectionTestId, setConnectionTestId] = useState<number>(0);
|
||||
const [testTargets, setTestTargets] = useState<AutoRouterTestTarget[]>([]);
|
||||
|
||||
const [selectedPreset, setSelectedPreset] = useState<string | undefined>(undefined);
|
||||
|
||||
useEffect(() => {
|
||||
const fetchModelAccessGroups = async () => {
|
||||
const response = await modelAvailableCall(accessToken, "", "", false, null, true, true);
|
||||
|
|
@ -81,8 +111,10 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
try {
|
||||
const uniqueModels = await fetchAvailableModels(accessToken);
|
||||
setModelInfo(uniqueModels);
|
||||
setModelsLoadState("loaded");
|
||||
} catch (error) {
|
||||
console.error("Error fetching model info for auto router:", error);
|
||||
setModelsLoadState("error");
|
||||
}
|
||||
};
|
||||
loadModels();
|
||||
|
|
@ -95,6 +127,69 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
label: model_group,
|
||||
}));
|
||||
|
||||
const availableModelSet = new Set(modelInfo.map((m) => m.model_group));
|
||||
const presets = getAllPresets();
|
||||
|
||||
// A preset's models can only be trusted against a successfully loaded list. Selection and the
|
||||
// greyed-out state derive from this one function, so a preset that cannot be selected can never
|
||||
// have been applied: while loading we withhold selection rather than let a caller pick a preset
|
||||
// whose models we cannot yet verify, and a failed fetch leaves every preset unverifiable. This
|
||||
// makes the load-race (pick during loading, then discover a missing model) unrepresentable.
|
||||
const presetAvailability = (preset: AutoRouterPreset): PresetAvailability => {
|
||||
if (modelsLoadState === "loading") return { kind: "loading" };
|
||||
if (modelsLoadState === "error") return { kind: "unverifiable" };
|
||||
const missing = getMissingModelsInPreset(preset, availableModelSet);
|
||||
return missing.length > 0 ? { kind: "missing_models", models: missing } : { kind: "available" };
|
||||
};
|
||||
|
||||
const resetToCustom = () => {
|
||||
setComplexityRouterConfig({
|
||||
tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
classifier_type: "heuristic",
|
||||
});
|
||||
setCustomTechnicalKeywords([]);
|
||||
setKeywordTierRules([]);
|
||||
setSemanticMatchingEnabled(false);
|
||||
setEmbeddingModel(undefined);
|
||||
setMatchThreshold(DEFAULT_MATCH_THRESHOLD);
|
||||
setEscalationKeywords(DEFAULT_ESCALATION_KEYWORDS);
|
||||
};
|
||||
|
||||
const handlePresetChange = (presetKey: string | undefined) => {
|
||||
if (!presetKey || presetKey === "custom") {
|
||||
setSelectedPreset(presetKey);
|
||||
resetToCustom();
|
||||
return;
|
||||
}
|
||||
|
||||
const preset = getPresetByKey(presetKey);
|
||||
// Refuse to apply a preset whose models are not verified available. The dropdown disables
|
||||
// these options, so this is a guard against a stale click resolving after the list changed.
|
||||
if (!preset || presetAvailability(preset).kind !== "available") return;
|
||||
|
||||
setSelectedPreset(presetKey);
|
||||
|
||||
const config = preset.complexity_router_config;
|
||||
const presetComplexityRouterConfig: ComplexityRouterConfigValue = {
|
||||
tiers: config.tiers,
|
||||
classifier_type: config.classifier_type,
|
||||
classifier_llm_config: config.classifier_llm_config,
|
||||
adaptive: config.adaptive,
|
||||
adaptive_weights: config.adaptive_weights,
|
||||
tier_distance_penalty: config.tier_distance_penalty,
|
||||
adaptive_eligible: config.adaptive_eligible,
|
||||
return_raw_model_name: config.return_raw_model_name,
|
||||
};
|
||||
setComplexityRouterConfig(presetComplexityRouterConfig);
|
||||
|
||||
setCustomTechnicalKeywords(config.custom_technical_keywords || []);
|
||||
setKeywordTierRules(hydrateKeywordTierRules(config.keyword_tier_rules || []));
|
||||
setSemanticMatchingEnabled(config.semantic_keyword_matching || false);
|
||||
setEmbeddingModel(config.embedding_model);
|
||||
setMatchThreshold(config.match_threshold || DEFAULT_MATCH_THRESHOLD);
|
||||
setEscalationKeywords(config.escalation_keywords || DEFAULT_ESCALATION_KEYWORDS);
|
||||
};
|
||||
|
||||
const submitRecommendedRouter = (name: string) => {
|
||||
const {
|
||||
tiers,
|
||||
|
|
@ -221,6 +316,49 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
wrapperCol={{ span: 16 }}
|
||||
labelAlign="left"
|
||||
>
|
||||
<div className="mb-6">
|
||||
<label className="block text-sm font-medium text-gray-900 mb-2">
|
||||
Template <span className="text-red-500">*</span>
|
||||
</label>
|
||||
<AntdSelect
|
||||
value={selectedPreset}
|
||||
onChange={handlePresetChange}
|
||||
placeholder="Choose a template or select Custom to define your own"
|
||||
className="w-full"
|
||||
optionLabelProp="label"
|
||||
data-testid="template-selector"
|
||||
>
|
||||
<AntdSelect.Option value="custom" label="Custom Configuration">
|
||||
<div>
|
||||
<div className="font-medium">Custom Configuration</div>
|
||||
<div className="text-xs text-gray-500">Define your auto router from scratch</div>
|
||||
</div>
|
||||
</AntdSelect.Option>
|
||||
{presets.map((preset) => {
|
||||
const availability = presetAvailability(preset);
|
||||
const disabledHint = presetDisabledHint(availability);
|
||||
const isDisabled = disabledHint !== null;
|
||||
const hintClass = isPresetHintAlarming(availability) ? "text-red-500" : "text-gray-400";
|
||||
|
||||
return (
|
||||
<AntdSelect.Option
|
||||
key={preset.key}
|
||||
value={preset.key}
|
||||
label={preset.label}
|
||||
disabled={isDisabled}
|
||||
title={disabledHint ?? preset.description}
|
||||
>
|
||||
<div>
|
||||
<div className="font-medium">{preset.label}</div>
|
||||
<div className="text-xs text-gray-500">{preset.description}</div>
|
||||
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
|
||||
</div>
|
||||
</AntdSelect.Option>
|
||||
);
|
||||
})}
|
||||
</AntdSelect>
|
||||
</div>
|
||||
|
||||
<Form.Item
|
||||
rules={[{ required: true, message: "Auto router name is required" }]}
|
||||
label="Auto Router Name"
|
||||
|
|
|
|||
54
ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
Normal file
54
ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
getAllPresets,
|
||||
getPresetByKey,
|
||||
getRequiredModelsInPreset,
|
||||
getMissingModelsInPreset,
|
||||
} from "./autorouter_presets";
|
||||
|
||||
describe("autorouter_presets", () => {
|
||||
it("loads exactly the two model-family presets (sample_spec excluded)", () => {
|
||||
const presets = getAllPresets();
|
||||
expect(presets.map((p) => p.label).sort()).toEqual(["Anthropic Family", "OpenAI Family"]);
|
||||
// Every preset carries all four fields the UI relies on; a JSON typo dropping one fails here.
|
||||
for (const p of presets) {
|
||||
expect(p).toMatchObject({ key: expect.any(String), label: expect.any(String), description: expect.any(String) });
|
||||
expect(p.complexity_router_config.tiers).toBeTruthy();
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves a preset by its stable JSON key, not its display label", () => {
|
||||
expect(getPresetByKey("anthropic_family")?.label).toBe("Anthropic Family");
|
||||
expect(getPresetByKey("does_not_exist")).toBeUndefined();
|
||||
// sample_spec is filtered out, so it is not resolvable by key either.
|
||||
expect(getPresetByKey("sample_spec")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps every preset a plain heuristic complexity router (no adaptive/quality settings)", () => {
|
||||
for (const { complexity_router_config: config } of getAllPresets()) {
|
||||
expect(config.classifier_type).toBe("heuristic");
|
||||
expect(config.adaptive).toBeUndefined();
|
||||
expect(config.adaptive_weights).toBeUndefined();
|
||||
expect(config.adaptive_eligible).toBeUndefined();
|
||||
expect(config.tier_distance_penalty).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("collects every tier model as a required model", () => {
|
||||
const preset = getPresetByKey("anthropic_family")!;
|
||||
const required = getRequiredModelsInPreset(preset);
|
||||
const tierModels = Object.values(preset.complexity_router_config.tiers).flat();
|
||||
expect(tierModels.length).toBeGreaterThan(0);
|
||||
for (const model of tierModels) expect(required.has(model)).toBe(true);
|
||||
});
|
||||
|
||||
it("reports only the models the caller is missing, and none when the family is fully available", () => {
|
||||
const preset = getPresetByKey("openai_family")!;
|
||||
const required = [...getRequiredModelsInPreset(preset)];
|
||||
|
||||
expect(getMissingModelsInPreset(preset, new Set(["gpt-5-nano"]))).toEqual(
|
||||
required.filter((m) => m !== "gpt-5-nano").sort(),
|
||||
);
|
||||
expect(getMissingModelsInPreset(preset, new Set(required))).toEqual([]);
|
||||
});
|
||||
});
|
||||
34
ui/litellm-dashboard/src/lib/autorouter_presets.ts
Normal file
34
ui/litellm-dashboard/src/lib/autorouter_presets.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import { ComplexityRouterConfigPayload } from "@/components/add_model/build_complexity_router_config";
|
||||
import presetsRaw from "@/autorouter_presets.json";
|
||||
|
||||
// `key` is the stable JSON object key (e.g. "anthropic_family"); `label` is display text and
|
||||
// never an identity.
|
||||
export interface AutoRouterPreset {
|
||||
key: string;
|
||||
label: string;
|
||||
description: string;
|
||||
complexity_router_config: ComplexityRouterConfigPayload;
|
||||
}
|
||||
|
||||
// The bundled JSON is a developer-authored, build-time asset, so it is trusted at the import
|
||||
// boundary rather than re-validated at runtime (resolveJsonModule widens its string literals,
|
||||
// hence this one cast). autorouter_presets.test.ts pins the parsed shape, so a JSON typo fails
|
||||
// CI. "sample_spec" documents the shape and is not a selectable preset.
|
||||
const RAW = presetsRaw as Record<string, Omit<AutoRouterPreset, "key">>;
|
||||
|
||||
const PRESETS: AutoRouterPreset[] = Object.entries(RAW)
|
||||
.filter(([key]) => key !== "sample_spec")
|
||||
.map(([key, preset]) => ({ key, ...preset }));
|
||||
|
||||
export const getAllPresets = (): AutoRouterPreset[] => PRESETS;
|
||||
|
||||
export const getPresetByKey = (key: string): AutoRouterPreset | undefined => PRESETS.find((p) => p.key === key);
|
||||
|
||||
export const getRequiredModelsInPreset = (preset: AutoRouterPreset): Set<string> => {
|
||||
const { tiers, classifier_llm_config: classifier, embedding_model: embedding } = preset.complexity_router_config;
|
||||
const models = [...tiers.SIMPLE, ...tiers.MEDIUM, ...tiers.COMPLEX, ...tiers.REASONING, classifier?.model, embedding];
|
||||
return new Set(models.filter((model): model is string => model != null));
|
||||
};
|
||||
|
||||
export const getMissingModelsInPreset = (preset: AutoRouterPreset, availableModels: Set<string>): string[] =>
|
||||
[...getRequiredModelsInPreset(preset)].filter((model) => !availableModels.has(model)).sort();
|
||||
Loading…
Add table
Reference in a new issue