scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-10 03:28:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 84

feat(ui): match auto-router preset models against deployments' underlying model IDs (#35972)

Browse source 
The bundled presets only became selectable when an admin's public model_group
names matched the preset's hardcoded model names. model_name is admin-arbitrary,
so renamed deployments (my-claude-fast, bedrock-opus) left both presets greyed
out. Resolve preset models against each deployment's litellm_params.model and
model_info.base_model from /v2/model/info via a normalized ID join, and prefill
the admin's registered group names. Resolves LIT-5225
This commit is contained in:
tin-berri 2026-08-05 15:24:54 -07:00 • committed by GitHub
parent f047124b5a
commit 265945dfcd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 413 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts
View file

@ -101,6 +101,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment {
id?: string | null;
/** False for config.yaml-defined deployments, which the update and delete routes refuse. */
db_model?: boolean | null;
base_model?: string | null;
created_at?: string | null;
updated_at?: string | null;
team_id?: string | null;
@ -122,7 +123,7 @@ export const selectAutoRouterModelGroups = (deployments: AutoRouterCandidateDepl
export const selectAutoRouterDeployments = (deployments: AutoRouterDeployment[]): AutoRouterDeployment[] =>
deployments.filter(isAutoRouterDeployment);
const fetchAllModelDeployments = async (
export const fetchAllModelDeployments = async (
accessToken: string,
userId: string,
userRole: string,
@ -151,7 +152,7 @@ const fetchAllModelDeployments = async (
* A private namespace meant an edit through ModelInfoView left this list stale, and every
* future writer would have had to remember a second key.
*/
const autoRouterListKey = (userId: string | null, userRole: string | null) =>
export const autoRouterListKey = (userId: string | null, userRole: string | null) =>
modelKeys.list({
filters: {
scope: "autoRouters",

1
ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx
View file

@ -104,6 +104,7 @@ export function AutoRoutersPanel({ accessToken, userRole, userID, teams, createS
handleOk={handleCreated}
accessToken={accessToken}
userRole={userRole}
userId={userID}
createScope={createScope}
/>
</DialogContent>

112
ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx
View file

@ -41,7 +41,10 @@ const optionByLabel = (label: string): HTMLElement | undefined =>
const isOptionDisabled = (option: HTMLElement): boolean => option.classList.contains("ant-select-item-option-disabled");
const { mockFetchAvailableModels } = vi.hoisted(() => ({ mockFetchAvailableModels: vi.fn() }));
const { mockFetchAvailableModels, mockFetchAllModelDeployments } = vi.hoisted(() => ({
mockFetchAvailableModels: vi.fn(),
mockFetchAllModelDeployments: vi.fn(),
}));
vi.mock("../networking", () => ({
modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }),
@ -52,6 +55,11 @@ vi.mock("@/components/llm_calls/fetch_models", () => ({
fetchAvailableModels: mockFetchAvailableModels,
}));
vi.mock("@/app/(dashboard)/hooks/models/useModels", async (importOriginal) => {
const actual = await importOriginal<typeof import("@/app/(dashboard)/hooks/models/useModels")>();
return { ...actual, fetchAllModelDeployments: mockFetchAllModelDeployments };
});
vi.mock("./handle_add_auto_router_submit", () => ({
handleAddAutoRouterSubmit: vi.fn(),
}));
@ -93,6 +101,7 @@ describe("AddAutoRouterTab", () => {
// test's data instead of its own mock).
testQueryClient.clear();
mockFetchAvailableModels.mockResolvedValue([]);
mockFetchAllModelDeployments.mockResolvedValue([]);
});
// Detailed Configuration starts collapsed so the modal opens onto just Name + Template; a caller
@ -429,6 +438,16 @@ describe("AddAutoRouterTab", () => {
expect(anthropicOption.textContent).toContain("Cannot verify these models are available");
});
it("keeps group-name presets selectable when only the deployment fetch fails", async () => {
mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS);
mockFetchAllModelDeployments.mockRejectedValue(new Error("network error"));
renderWithProviders(<Harness />);
await waitForPresetEnabled("Anthropic Family");
await waitForPresetEnabled("OpenAI Family");
});
it("disables a preset missing one of its models, naming the missing model", async () => {
mockFetchAvailableModels.mockResolvedValue(
ALL_FAMILY_MODELS.filter((m) => m.model_group !== ANTHROPIC_ONLY_MODEL),
@ -540,4 +559,95 @@ describe("AddAutoRouterTab", () => {
expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled();
});
});
describe("deployment-matched presets", () => {
const renamedDeploymentsFor = (presetKey: string) =>
[...getRequiredModelsInPreset(getPresetByKey(presetKey)!)].map((model, index) => ({
model_name: `renamed-${presetKey}-${index}`,
litellm_params: { model: `someprovider/${model}` },
}));
const groupsFor = (deployments: { model_name: string }[]): ModelGroup[] =>
deployments.map((deployment) => ({ model_group: deployment.model_name, mode: "chat" }));
const ALL_RENAMED_DEPLOYMENTS = getAllPresets().flatMap((preset) => renamedDeploymentsFor(preset.key));
const renamedGroupFor = (model: string): string =>
ALL_RENAMED_DEPLOYMENTS.find((deployment) => deployment.litellm_params.model === `someprovider/${model}`)!
.model_name;
it("enables a preset whose models exist only under renamed deployments, labeling the match", async () => {
mockFetchAvailableModels.mockResolvedValue(groupsFor(ALL_RENAMED_DEPLOYMENTS));
mockFetchAllModelDeployments.mockResolvedValue(ALL_RENAMED_DEPLOYMENTS);
renderWithProviders(<Harness />);
openTemplateDropdown();
await waitFor(() => {
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false);
});
expect(optionByLabel("Anthropic Family")!.textContent).toContain("Matches your deployments");
});
it("keeps detailed configuration open and prefills the admin's group names on apply", async () => {
const user = userEvent.setup();
mockFetchAvailableModels.mockResolvedValue(groupsFor(ALL_RENAMED_DEPLOYMENTS));
mockFetchAllModelDeployments.mockResolvedValue(ALL_RENAMED_DEPLOYMENTS);
renderWithProviders(<Harness />);
openTemplateDropdown();
await waitFor(() => {
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false);
});
fireEvent.click(optionByLabel("Anthropic Family")!);
expect(screen.getByText("Advanced: Keyword/Semantic Matching")).toBeInTheDocument();
await user.type(screen.getByPlaceholderText(/smart_router/i), "renamed-router");
await user.click(screen.getByRole("button", { name: /add auto router/i }));
await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled());
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({
complexity_router_config: {
tiers: {
SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(renamedGroupFor),
MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(renamedGroupFor),
COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(renamedGroupFor),
REASONING: ANTHROPIC_TIERS.REASONING.map(renamedGroupFor),
},
},
});
});
it("lists a deployment-matched preset ahead of one that stays unavailable", async () => {
const anthropicOnly = renamedDeploymentsFor("anthropic_family");
mockFetchAvailableModels.mockResolvedValue(groupsFor(anthropicOnly));
mockFetchAllModelDeployments.mockResolvedValue(anthropicOnly);
renderWithProviders(<Harness />);
openTemplateDropdown();
await waitFor(() => {
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false);
});
const labels = Array.from(document.querySelectorAll<HTMLElement>(".ant-select-item-option")).map(
(option) => option.querySelector(".font-medium")?.textContent,
);
expect(labels).toEqual(["Anthropic Family", "OpenAI Family", "Custom Configuration"]);
});
it("never lets a wildcard deployment satisfy a preset", async () => {
const wildcard = [{ model_name: "openai-wild", litellm_params: { model: "openai/*" } }];
mockFetchAvailableModels.mockResolvedValue(groupsFor(wildcard));
mockFetchAllModelDeployments.mockResolvedValue(wildcard);
renderWithProviders(<Harness />);
openTemplateDropdown();
await waitFor(() => {
expect(optionByLabel("OpenAI Family")!.textContent).toContain("Missing:");
});
expect(isOptionDisabled(optionByLabel("OpenAI Family")!)).toBe(true);
});
});
});

77
ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
View file

@ -9,6 +9,7 @@ import { type ModelWriteScope } from "@/utils/modelPermissions";
import TeamDropdown from "../common_components/team_dropdown";
import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
import { fetchAvailableModels } from "@/components/llm_calls/fetch_models";
import { autoRouterListKey, fetchAllModelDeployments } from "@/app/(dashboard)/hooks/models/useModels";
import ComplexityRouterConfig, {
ComplexityRouterConfigValue,
ComplexityTiers,
@ -38,6 +39,9 @@ import {
getReferencedModelsError,
buildEmptyPrefill,
buildPresetPrefill,
buildModelAvailability,
deploymentRefsFromModelInfo,
ModelAvailability,
PresetPrefill,
AutoRouterPreset,
} from "@/lib/autorouter_presets";
@ -46,6 +50,7 @@ interface AddAutoRouterTabProps {
handleOk: () => void;
accessToken: string;
userRole: string;
userId?: string | null;
/**
* How this caller must scope what they create. A team admin has to name a team, because
* POST /model/new rejects an unscoped create from any non-proxy-admin; without the selector
@ -55,7 +60,7 @@ interface AddAutoRouterTabProps {
}
type PresetAvailability =
| { kind: "available" }
| { kind: "available"; viaDeployments: boolean }
| { kind: "loading" }
| { kind: "unverifiable" }
| { kind: "missing_models"; models: readonly string[] };
@ -110,17 +115,18 @@ const getSubmitBlockedReason = (
config: ComplexityRouterConfigValue,
keywordTierRules: KeywordTierRule[],
referencedModelsParams: Parameters<typeof getReferencedModelsError>[0],
availableModelSet: Set<string>,
availability: ModelAvailability,
): string | null =>
getMissingTiersError(config.tiers) ??
getTierLabelsError(config.tier_labels) ??
getKeywordTierRulesError(keywordTierRules) ??
getReferencedModelsError(referencedModelsParams, availableModelSet);
getReferencedModelsError(referencedModelsParams, availability);
const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
handleOk,
accessToken,
userRole,
userId,
createScope = "unscoped-ok",
}) => {
const requiresTeamScope = createScope === "team-required";
@ -163,7 +169,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
const {
data,
isLoading: modelsLoading,
isLoading: groupsLoading,
isError: modelsError,
refetch: refetchModels,
} = useQuery({
@ -171,6 +177,12 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
queryFn: () => fetchAvailableModels(accessToken),
enabled: Boolean(accessToken),
});
const { data: deployments, isLoading: deploymentsLoading } = useQuery({
queryKey: autoRouterListKey(userId ?? "", userRole),
queryFn: () => fetchAllModelDeployments(accessToken, userId ?? "", userRole),
enabled: Boolean(accessToken),
});
const modelsLoading = groupsLoading || deploymentsLoading;
const modelInfo = React.useMemo(() => data ?? [], [data]);
// react-query keeps the last successful list around when a later refetch fails, so isError alone
// can't tell "never loaded" apart from "loaded, then a background refetch errored" - only the
@ -184,7 +196,22 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
label: model_group,
}));
const availableModelSet = React.useMemo(() => new Set(modelInfo.map((m) => m.model_group)), [modelInfo]);
const availability = React.useMemo(
() =>
buildModelAvailability(
modelInfo.map((m) => m.model_group),
deploymentRefsFromModelInfo(deployments ?? []),
),
[modelInfo, deployments],
);
const groupsOnlyAvailability = React.useMemo(
() =>
buildModelAvailability(
modelInfo.map((m) => m.model_group),
[],
),
[modelInfo],
);
// A preset's models can only be trusted against a successfully loaded list. Selection and the
// greyed-out state derive from this one function, so a preset that cannot be selected can never
@ -195,10 +222,22 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
(preset: AutoRouterPreset): PresetAvailability => {
if (modelsLoading) return { kind: "loading" };
if (modelsUnverifiable) return { kind: "unverifiable" };
const missing = getMissingModelsInPreset(preset, availableModelSet);
return missing.length > 0 ? { kind: "missing_models", models: missing } : { kind: "available" };
const missing = getMissingModelsInPreset(preset, availability);
if (missing.length > 0) return { kind: "missing_models", models: missing };
return {
kind: "available",
viaDeployments: getMissingModelsInPreset(preset, groupsOnlyAvailability).length > 0,
};
},
[modelsLoading, modelsUnverifiable, availableModelSet],
[modelsLoading, modelsUnverifiable, availability, groupsOnlyAvailability],
);
const sortedPresetOptions = React.useMemo(
() =>
presets
.map((preset) => ({ preset, availability: presetAvailability(preset) }))
.sort((a, b) => Number(b.availability.kind === "available") - Number(a.availability.kind === "available")),
[presetAvailability],
);
const applyPrefill = (prefill: PresetPrefill) => {
@ -222,11 +261,13 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
const preset = getPresetByKey(presetKey);
// Refuse to apply a preset whose models are not verified available. The dropdown disables
// these options, so this is a guard against a stale click resolving after the list changed.
if (!preset || presetAvailability(preset).kind !== "available") return;
if (!preset) return;
const presetState = presetAvailability(preset);
if (presetState.kind !== "available") return;
setSelectedPreset(presetKey);
applyPrefill(buildPresetPrefill(preset.complexity_router_config, availableModelSet));
setDetailsExpanded(false);
applyPrefill(buildPresetPrefill(preset.complexity_router_config, availability));
setDetailsExpanded(presetState.viaDeployments);
};
const referencedModelsParams = {
@ -241,7 +282,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
complexityRouterConfig,
keywordTierRules,
referencedModelsParams,
availableModelSet,
groupsOnlyAvailability,
);
const complexityRouterConfigParams: BuildComplexityRouterConfigParams = {
@ -308,7 +349,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
// same handler) fires on Enter regardless of the button's disabled state - without this check,
// Enter in the name field could still create a router referencing a model that disappeared from
// availableModelSet after the tiers were filled in.
const referencedModelsError = getReferencedModelsError(referencedModelsParams, availableModelSet);
const referencedModelsError = getReferencedModelsError(referencedModelsParams, groupsOnlyAvailability);
if (referencedModelsError) {
setShowValidationErrors(true);
NotificationManager.fromBackend(referencedModelsError);
@ -405,11 +446,12 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
optionLabelProp="label"
data-testid="template-selector"
>
{presets.map((preset) => {
const availability = presetAvailability(preset);
const disabledHint = presetDisabledHint(availability);
{sortedPresetOptions.map(({ preset, availability: presetState }) => {
const disabledHint = presetDisabledHint(presetState);
const isDisabled = disabledHint !== null;
const hintClass = isPresetHintAlarming(availability) ? "text-red-500" : "text-gray-400";
const hintClass = isPresetHintAlarming(presetState) ? "text-red-500" : "text-gray-400";
const matchedHint =
presetState.kind === "available" && presetState.viaDeployments ? "Matches your deployments" : null;
return (
<AntdSelect.Option
@ -423,6 +465,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
<div className="font-medium">{preset.label}</div>
<div className="text-xs text-gray-500">{preset.description}</div>
{disabledHint && <div className={`text-xs mt-1 ${hintClass}`}>{disabledHint}</div>}
{matchedHint && <div className="text-xs mt-1 text-green-600">{matchedHint}</div>}
</div>
</AntdSelect.Option>
);

170
ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
View file

@ -9,10 +9,14 @@ import {
getReferencedModelsError,
buildEmptyPrefill,
buildPresetPrefill,
buildModelAvailability,
deploymentRefsFromModelInfo,
} from "./autorouter_presets";
import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching";
import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords";
const groupsOnly = (models: Iterable<string>) => buildModelAvailability(models, []);
describe("autorouter_presets", () => {
it("loads exactly the two model-family presets", () => {
const presets = getAllPresets();
@ -52,8 +56,8 @@ describe("autorouter_presets", () => {
const required = [...getRequiredModelsInPreset(preset)];
const [held] = required;
expect(getMissingModelsInPreset(preset, new Set([held]))).toEqual(required.filter((m) => m !== held).sort());
expect(getMissingModelsInPreset(preset, new Set(required))).toEqual([]);
expect(getMissingModelsInPreset(preset, groupsOnly([held]))).toEqual(required.filter((m) => m !== held).sort());
expect(getMissingModelsInPreset(preset, groupsOnly(required))).toEqual([]);
});
// Admins spell version numbers with either "-" or "." (claude-sonnet-4-5 vs claude-sonnet-4.5);
@ -67,14 +71,14 @@ describe("autorouter_presets", () => {
const dottedSpellings = required.map((model) => model.replace(/(\d)-(\d)/g, "$1.$2"));
expect(dottedSpellings).not.toEqual(required);
expect(getMissingModelsInPreset(preset, new Set(dottedSpellings))).toEqual([]);
expect(getMissingModelsInPreset(preset, groupsOnly(dottedSpellings))).toEqual([]);
});
// The two-arm mirror: a differently-punctuated preset model must not be reported missing.
it("does not flag a differently-punctuated model as missing via getMissingModels directly", () => {
const missing = getMissingModels(
{ tiers: { SIMPLE: ["claude-sonnet-4-5"], MEDIUM: [], COMPLEX: [], REASONING: [] } },
new Set(["claude-sonnet-4.5"]),
groupsOnly(["claude-sonnet-4.5"]),
);
expect(missing).toEqual([]);
});
@ -90,9 +94,150 @@ describe("autorouter_presets", () => {
expect(required).toEqual(new Set(["gpt-5-nano"]));
});
describe("deployment matching (underlying provider model IDs)", () => {
const availabilityFor = (modelGroup: string, underlyingModel: string) =>
buildModelAvailability([modelGroup], [{ modelGroup, underlyingModels: [underlyingModel] }]);
it.each([
["provider prefix", "my-claude-fast", "anthropic/claude-haiku-4-5", "claude-haiku-4-5"],
["bedrock region+namespace+revision", "bedrock-opus", "bedrock/us.anthropic.claude-opus-5-v1:0", "claude-opus-5"],
["bedrock date stamp", "bedrock-opus41", "bedrock/us.anthropic.claude-opus-4-1-20250805-v1:0", "claude-opus-4-1"],
["dotted version", "team-gpt", "openai/gpt-5.4", "gpt-5.4"],
["vertex @tag", "vertex-frontier", "vertex_ai/claude-fable-5@default", "claude-fable-5"],
["bedrock 1m context label", "opus-1m", "bedrock/us.anthropic.claude-opus-5-v1:0[1m]", "claude-opus-5"],
])("resolves a %s deployment and prefills the admin's group name", (_label, group, underlying, presetModel) => {
const availability = availabilityFor(group, underlying);
const config = {
tiers: { SIMPLE: [presetModel], MEDIUM: [], COMPLEX: [], REASONING: [] },
classifier_type: "heuristic" as const,
session_affinity: false,
};
expect(getMissingModels(config, availability)).toEqual([]);
expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual([group]);
});
it.each([
["gpt-5.4", "openai/gpt-5.4-mini"],
["gpt-5.4-mini", "openai/gpt-5.4"],
["o3", "openai/o3-mini"],
["o3-mini", "openai/o3"],
["some-model", "prov/some-model-20991399"],
])("never lets %s be satisfied by a deployment of %s", (presetModel, underlying) => {
const availability = availabilityFor("some-group", underlying);
const config = { tiers: { SIMPLE: [presetModel], MEDIUM: [], COMPLEX: [], REASONING: [] } };
expect(getMissingModels(config, availability)).toEqual([presetModel]);
});
it("never indexes a wildcard deployment", () => {
const availability = availabilityFor("openai-wild", "openai/*");
expect(availability.underlyingIndex.size).toBe(0);
});
it("ignores a deployment whose group is not itself an available model group", () => {
const availability = buildModelAvailability(
["some-other-group"],
[{ modelGroup: "orphan-group", underlyingModels: ["anthropic/claude-opus-5"] }],
);
expect(availability.underlyingIndex.size).toBe(0);
});
it("breaks ties between groups serving the same model deterministically, alphabetically", () => {
const availability = buildModelAvailability(
["z-group", "a-group"],
[
{ modelGroup: "z-group", underlyingModels: ["anthropic/claude-opus-5"] },
{ modelGroup: "a-group", underlyingModels: ["bedrock/us.anthropic.claude-opus-5-v1:0"] },
],
);
const config = {
tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] },
classifier_type: "heuristic" as const,
session_affinity: false,
};
expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual(["a-group"]);
});
it("prefers an exact group-name match over the deployment index", () => {
const availability = buildModelAvailability(
["claude-opus-5", "renamed-opus"],
[{ modelGroup: "renamed-opus", underlyingModels: ["anthropic/claude-opus-5"] }],
);
const config = {
tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] },
classifier_type: "heuristic" as const,
session_affinity: false,
};
expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual(["claude-opus-5"]);
});
it.each(getAllPresets().map((preset) => [preset.key, preset] as const))(
"fully resolves the %s preset through renamed deployments only",
(_key, preset) => {
const required = [...getRequiredModelsInPreset(preset)];
const groups = required.map((_model, index) => `renamed-${index}`);
const availability = buildModelAvailability(
groups,
required.map((model, index) => ({
modelGroup: `renamed-${index}`,
underlyingModels: [`someprovider/${model}`],
})),
);
expect(getMissingModelsInPreset(preset, availability)).toEqual([]);
const prefilled = buildPresetPrefill(preset.complexity_router_config, availability);
const prefilledModels = Object.values(prefilled.complexityRouterConfig.tiers).flat();
expect(prefilledModels.length).toBeGreaterThan(0);
for (const model of prefilledModels) expect(groups).toContain(model);
},
);
});
describe("deploymentRefsFromModelInfo", () => {
it("keeps litellm_params.model and model_info.base_model, drops rows with neither or no name", () => {
const refs = deploymentRefsFromModelInfo([
{
model_name: "azure-prod",
litellm_params: { model: "azure/my-deployment" },
model_info: { base_model: "azure/gpt-5.4" },
},
{ model_name: "no-underlying", litellm_params: {}, model_info: {} },
{ litellm_params: { model: "openai/gpt-5.4" } },
]);
expect(refs).toEqual([{ modelGroup: "azure-prod", underlyingModels: ["azure/my-deployment", "azure/gpt-5.4"] }]);
});
it("lets an azure deployment resolve through base_model declared under litellm_params", () => {
const availability = buildModelAvailability(
["azure-lp"],
deploymentRefsFromModelInfo([
{
model_name: "azure-lp",
litellm_params: { model: "azure/opaque-deployment-name", base_model: "azure/gpt-5.4" },
},
]),
);
const config = { tiers: { SIMPLE: ["gpt-5.4"], MEDIUM: [], COMPLEX: [], REASONING: [] } };
expect(getMissingModels(config, availability)).toEqual([]);
});
it("lets an azure deployment resolve through its admin-declared base_model", () => {
const availability = buildModelAvailability(
["azure-prod"],
deploymentRefsFromModelInfo([
{
model_name: "azure-prod",
litellm_params: { model: "azure/opaque-deployment-name" },
model_info: { base_model: "azure/gpt-5.4" },
},
]),
);
const config = { tiers: { SIMPLE: ["gpt-5.4"], MEDIUM: [], COMPLEX: [], REASONING: [] } };
expect(getMissingModels(config, availability)).toEqual([]);
});
});
describe("getReferencedModelsError", () => {
const tiers = { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] };
const available = new Set(["gpt-5-nano"]);
const available = groupsOnly(["gpt-5-nano"]);
// Both fields are always populated with a model missing from `available`; only the
// enabled/disabled toggles below decide whether that missing model gets reported.
const params = {
@ -137,7 +282,10 @@ describe("autorouter_presets", () => {
describe("buildPresetPrefill", () => {
it("prefills a real bundled preset's tiers into the config", () => {
const preset = getPresetByKey("anthropic_family")!;
const prefill = buildPresetPrefill(preset.complexity_router_config, getRequiredModelsInPreset(preset));
const prefill = buildPresetPrefill(
preset.complexity_router_config,
groupsOnly(getRequiredModelsInPreset(preset)),
);
expect(prefill.complexityRouterConfig.tiers).toEqual(preset.complexity_router_config.tiers);
});
@ -153,7 +301,7 @@ describe("autorouter_presets", () => {
match_threshold: 0,
escalation_keywords: [],
};
const prefill = buildPresetPrefill(config, new Set(["gpt-5-nano"]));
const prefill = buildPresetPrefill(config, groupsOnly(["gpt-5-nano"]));
expect(prefill.matchThreshold).toBe(0);
expect(prefill.escalationKeywords).toEqual([]);
});
@ -165,7 +313,7 @@ describe("autorouter_presets", () => {
classifier_type: "heuristic",
session_affinity: false,
},
new Set(["gpt-5-nano"]),
groupsOnly(["gpt-5-nano"]),
);
expect(prefill.matchThreshold).toBe(DEFAULT_MATCH_THRESHOLD);
expect(prefill.escalationKeywords).toEqual(DEFAULT_ESCALATION_KEYWORDS);
@ -182,10 +330,10 @@ describe("autorouter_presets", () => {
};
const labeled = buildPresetPrefill(
{ ...base, tier_labels: { SIMPLE: "Cheap", REASONING: "Deep" } },
new Set(["gpt-5-nano"]),
groupsOnly(["gpt-5-nano"]),
);
expect(labeled.complexityRouterConfig.tier_labels).toEqual({ SIMPLE: "Cheap", REASONING: "Deep" });
expect(buildPresetPrefill(base, new Set(["gpt-5-nano"])).complexityRouterConfig.tier_labels).toBeUndefined();
expect(buildPresetPrefill(base, groupsOnly(["gpt-5-nano"])).complexityRouterConfig.tier_labels).toBeUndefined();
});
it("rewrites a preset's model name to the caller's differently-punctuated registered spelling", () => {
@ -194,7 +342,7 @@ describe("autorouter_presets", () => {
classifier_type: "heuristic" as const,
session_affinity: false,
};
const prefill = buildPresetPrefill(config, new Set(["claude-sonnet-4.5"]));
const prefill = buildPresetPrefill(config, groupsOnly(["claude-sonnet-4.5"]));
expect(prefill.complexityRouterConfig.tiers.SIMPLE).toEqual(["claude-sonnet-4.5"]);
});
});

95
ui/litellm-dashboard/src/lib/autorouter_presets.ts
View file

@ -55,27 +55,90 @@ export const getRequiredModels = (
// only the punctuation within one version number does.
const normalizeModelName = (model: string): string => model.replace(/(\d)\.(\d)/g, "$1-$2");
// The caller's actual registered spelling for a required model, under either separator
// convention, or undefined if truly absent. Preset prefill must write THIS spelling, not the
// preset's literal string - otherwise a caller whose proxy only has the dotted form ends up with
// a tier pointing at a model name that was never registered.
const resolveAvailableModel = (requiredModel: string, availableModels: Set<string>): string | undefined => {
if (availableModels.has(requiredModel)) return requiredModel;
export interface DeploymentModelRef {
modelGroup: string;
underlyingModels: readonly string[];
}
export interface ModelAvailability {
modelGroups: Set<string>;
underlyingIndex: Map<string, readonly string[]>;
}
const normalizeUnderlyingModel = (model: string): string | null => {
if (model.includes("*")) return null;
const ownName = model.slice(model.lastIndexOf("/") + 1).split("@")[0];
const lastNamespaceSegment = normalizeModelName(ownName).split(".").at(-1) ?? "";
const stripped = lastNamespaceSegment
.replace(/:\d+k$/i, "")
.replace(/\[\w+\]$/, "")
.replace(/-v\d+(:\d+)?$/, "")
.replace(/-20\d{2}(0[1-9]|1[0-2])(0[1-9]|[12]\d|3[01])$/, "");
return stripped.toLowerCase() || null;
};
export const buildModelAvailability = (
modelGroups: Iterable<string>,
deployments: readonly DeploymentModelRef[],
): ModelAvailability => {
const groups = new Set(modelGroups);
const entries = deployments
.filter((deployment) => groups.has(deployment.modelGroup))
.flatMap((deployment) =>
deployment.underlyingModels
.map(normalizeUnderlyingModel)
.filter((key): key is string => key !== null)
.map((key) => ({ key, modelGroup: deployment.modelGroup })),
);
const grouped = new Map<string, Set<string>>();
for (const entry of entries) {
const groupsForKey = grouped.get(entry.key) ?? new Set<string>();
groupsForKey.add(entry.modelGroup);
grouped.set(entry.key, groupsForKey);
}
const underlyingIndex = new Map(
Array.from(grouped, ([key, groupsForKey]) => [key, Array.from(groupsForKey).sort()] as const),
);
return { modelGroups: groups, underlyingIndex };
};
export const deploymentRefsFromModelInfo = (
rows: readonly {
model_name?: string | null;
litellm_params?: { model?: string | null; base_model?: string | null } | null;
model_info?: { base_model?: string | null } | null;
}[],
): DeploymentModelRef[] =>
rows.flatMap((row) => {
const underlyingModels = [
row.litellm_params?.model,
row.litellm_params?.base_model,
row.model_info?.base_model,
].filter((model): model is string => Boolean(model));
return row.model_name && underlyingModels.length > 0 ? [{ modelGroup: row.model_name, underlyingModels }] : [];
});
const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => {
const { modelGroups, underlyingIndex } = availability;
if (modelGroups.has(requiredModel)) return requiredModel;
const normalized = normalizeModelName(requiredModel);
return Array.from(availableModels).find((available) => normalizeModelName(available) === normalized);
const groupMatch = Array.from(modelGroups).find((available) => normalizeModelName(available) === normalized);
if (groupMatch !== undefined) return groupMatch;
const key = normalizeUnderlyingModel(requiredModel);
return key === null ? undefined : underlyingIndex.get(key)?.[0];
};
export const getMissingModels = (
config: Pick<ComplexityRouterConfigPayload, "tiers" | "classifier_llm_config" | "embedding_model">,
availableModels: Set<string>,
availability: ModelAvailability,
): string[] =>
[...getRequiredModels(config)].filter((model) => resolveAvailableModel(model, availableModels) === undefined).sort();
[...getRequiredModels(config)].filter((model) => resolveAvailableModel(model, availability) === undefined).sort();
export const getRequiredModelsInPreset = (preset: AutoRouterPreset): Set<string> =>
getRequiredModels(preset.complexity_router_config);
export const getMissingModelsInPreset = (preset: AutoRouterPreset, availableModels: Set<string>): string[] =>
getMissingModels(preset.complexity_router_config, availableModels);
export const getMissingModelsInPreset = (preset: AutoRouterPreset, availability: ModelAvailability): string[] =>
getMissingModels(preset.complexity_router_config, availability);
// Checks the config actually being built (whether it arrived via a preset prefill or was typed by
// hand - the two are indistinguishable once the caller has started editing), not a preset's
@ -91,7 +154,7 @@ export const getReferencedModelsError = (
semanticMatchingEnabled: boolean;
embeddingModel: string | undefined;
},
availableModels: Set<string>,
availability: ModelAvailability,
): string | null => {
const missing = getMissingModels(
{
@ -99,7 +162,7 @@ export const getReferencedModelsError = (
classifier_llm_config: params.classifierType === "llm" ? params.classifierLlmConfig : undefined,
embedding_model: params.semanticMatchingEnabled ? params.embeddingModel : undefined,
},
availableModels,
availability,
);
return missing.length > 0 ? `Model(s) no longer available: ${missing.join(", ")}` : null;
};
@ -133,16 +196,16 @@ export const buildEmptyPrefill = (): PresetPrefill => ({
// `??`, never `||`: a preset's match_threshold: 0 or escalation_keywords: [] is a deliberate,
// falsy value that must survive the prefill, not get silently replaced by the default.
//
// `availableModels` is required, not optional: every model reference gets rewritten to the
// `availability` is required, not optional: every model reference gets rewritten to the
// caller's actual registered spelling (resolveAvailableModel), which may differ from the preset's
// literal string by version-separator punctuation alone. Called only after presetAvailability has
// already confirmed every required model resolves, so falling back to the preset's own string
// when a model somehow doesn't resolve is unreachable in practice, not a silent-failure path.
export const buildPresetPrefill = (
config: ComplexityRouterConfigPayload,
availableModels: Set<string>,
availability: ModelAvailability,
): PresetPrefill => {
const resolve = (model: string): string => resolveAvailableModel(model, availableModels) ?? model;
const resolve = (model: string): string => resolveAvailableModel(model, availability) ?? model;
const resolveTier = (models: string[]): string[] => models.map(resolve);
return {