scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge pull request #35882 from BerriAI/litellm_fix_autorouter_preset_test_fixtures

Browse source 
fix(ui): derive auto-router preset tests from the bundled preset JSON
This commit is contained in:
yuneng-jiang 2026-08-04 19:13:36 -07:00 • committed by GitHub
commit fa2d97f7bc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 32 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ui/litellm-dashboard/src/autorouter_presets.json
View file

@ -16,7 +16,7 @@
},
"openai_family": {
"label": "OpenAI Family",
"description": "Routes across the GPT model family: gpt-5-nano for simple queries, gpt-5-mini for medium, gpt-5 for complex, o3 for reasoning-heavy requests.",
"description": "Routes across the GPT model family: gpt-5.4-nano for simple queries, gpt-5.4-mini for medium, gpt-5.4 for complex, o3 for reasoning-heavy requests.",
"complexity_router_config": {
"tiers": {
"SIMPLE": ["gpt-5.4-nano"],

42
ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx
View file

@ -7,18 +7,20 @@ import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
import { getMissingTiersError } from "./build_complexity_router_config";
import { testAutoRouterRouting } from "../networking";
import { ModelGroup } from "@/components/llm_calls/fetch_models";
import { getAllPresets, getPresetByKey, getRequiredModelsInPreset } from "@/lib/autorouter_presets";
// Every model referenced by both bundled family presets. A caller holding all of these can select
// either preset; dropping any one greys out the preset that names it.
const ANTHROPIC_PRESET = getPresetByKey("anthropic_family")!;
const ANTHROPIC_TIERS = ANTHROPIC_PRESET.complexity_router_config.tiers;
// Every model referenced by the bundled family presets, derived from the presets themselves so
// that renaming a preset's models in autorouter_presets.json does not red these tests. A caller
// holding all of these can select either preset; dropping any one greys out the preset that
// names it.
const ALL_FAMILY_MODELS: ModelGroup[] = [
{ model_group: "claude-haiku-4-5", mode: "chat" },
{ model_group: "claude-sonnet-4-5", mode: "chat" },
{ model_group: "claude-opus-5", mode: "chat" },
{ model_group: "gpt-5-nano", mode: "chat" },
{ model_group: "gpt-5-mini", mode: "chat" },
{ model_group: "gpt-5", mode: "chat" },
{ model_group: "o3", mode: "chat" },
];
...new Set(getAllPresets().flatMap((preset) => [...getRequiredModelsInPreset(preset)])),
].map((model_group) => ({ model_group, mode: "chat" }));
const ANTHROPIC_ONLY_MODEL = ANTHROPIC_TIERS.COMPLEX[0];
const openTemplateDropdown = (): void => {
fireEvent.mouseDown(screen.getByTestId("template-selector").querySelector(".ant-select-selector")!);
@ -428,13 +430,15 @@ describe("AddAutoRouterTab", () => {
});
it("disables a preset missing one of its models, naming the missing model", async () => {
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS.filter((m) => m.model_group !== "claude-opus-5"));
mockFetchAvailableModels.mockResolvedValue(
ALL_FAMILY_MODELS.filter((m) => m.model_group !== ANTHROPIC_ONLY_MODEL),
);
renderWithProviders(<Harness />);
openTemplateDropdown();
await waitFor(() => {
expect(optionByLabel("Anthropic Family")!.textContent).toContain("Missing: claude-opus-5");
expect(optionByLabel("Anthropic Family")!.textContent).toContain(`Missing: ${ANTHROPIC_ONLY_MODEL}`);
});
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(true);
});
@ -458,7 +462,8 @@ describe("AddAutoRouterTab", () => {
expect(screen.queryByText("Advanced: Keyword/Semantic Matching")).not.toBeInTheDocument();
expect(
screen.getByText(
"Simple: claude-haiku-4-5 · Medium: claude-sonnet-4-5 · Complex: claude-opus-5 · Reasoning: claude-opus-5",
`Simple: ${ANTHROPIC_TIERS.SIMPLE.join(", ")} · Medium: ${ANTHROPIC_TIERS.MEDIUM.join(", ")} · ` +
`Complex: ${ANTHROPIC_TIERS.COMPLEX.join(", ")} · Reasoning: ${ANTHROPIC_TIERS.REASONING.join(", ")}`,
),
).toBeInTheDocument();
});
@ -500,15 +505,8 @@ describe("AddAutoRouterTab", () => {
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({
auto_router_default_model: "claude-sonnet-4-5",
complexity_router_config: {
tiers: {
SIMPLE: ["claude-haiku-4-5"],
MEDIUM: ["claude-sonnet-4-5"],
COMPLEX: ["claude-opus-5"],
REASONING: ["claude-opus-5"],
},
},
auto_router_default_model: ANTHROPIC_TIERS.MEDIUM[0],
complexity_router_config: { tiers: ANTHROPIC_TIERS },
});
});

18
ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
View file

@ -50,20 +50,24 @@ describe("autorouter_presets", () => {
it("reports only the models the caller is missing, and none when the family is fully available", () => {
const preset = getPresetByKey("openai_family")!;
const required = [...getRequiredModelsInPreset(preset)];
const [held] = required;
expect(getMissingModelsInPreset(preset, new Set(["gpt-5-nano"]))).toEqual(
required.filter((m) => m !== "gpt-5-nano").sort(),
);
expect(getMissingModelsInPreset(preset, new Set([held]))).toEqual(required.filter((m) => m !== held).sort());
expect(getMissingModelsInPreset(preset, new Set(required))).toEqual([]);
});
// Admins spell version numbers with either "-" or "." (claude-sonnet-4-5 vs claude-sonnet-4.5);
// a caller who only registered one form still satisfies a preset that names the other.
// a caller who only registered one form still satisfies a preset that names the other. The
// caller's spellings are derived from the preset itself so that renaming a preset's models in
// autorouter_presets.json can't quietly turn this into a no-op (the inequality below fails
// instead, if no preset model carries a version number at all).
it("treats a preset's model as available under either version-separator spelling", () => {
const preset = getPresetByKey("anthropic_family")!;
expect(
getMissingModelsInPreset(preset, new Set(["claude-haiku-4.5", "claude-sonnet-4.5", "claude-opus-5"])),
).toEqual([]);
const required = [...getRequiredModelsInPreset(preset)];
const dottedSpellings = required.map((model) => model.replace(/(\d)-(\d)/g, "$1.$2"));
expect(dottedSpellings).not.toEqual(required);
expect(getMissingModelsInPreset(preset, new Set(dottedSpellings))).toEqual([]);
});
// The two-arm mirror: a differently-punctuated preset model must not be reported missing.