From 265945dfcdad01c62410c3660590d30ce3dff5ee Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 5 Aug 2026 15:24:54 -0700 Subject: [PATCH] feat(ui): match auto-router preset models against deployments' underlying model IDs (#35972) The bundled presets only became selectable when an admin's public model_group names matched the preset's hardcoded model names. model_name is admin-arbitrary, so renamed deployments (my-claude-fast, bedrock-opus) left both presets greyed out. Resolve preset models against each deployment's litellm_params.model and model_info.base_model from /v2/model/info via a normalized ID join, and prefill the admin's registered group names. Resolves LIT-5225 --- .../app/(dashboard)/hooks/models/useModels.ts | 5 +- .../AutoRouters/AutoRoutersPanel.tsx | 1 + .../add_model/add_auto_router_tab.test.tsx | 112 +++++++++++- .../add_model/add_auto_router_tab.tsx | 77 ++++++-- .../src/lib/autorouter_presets.test.ts | 170 ++++++++++++++++-- .../src/lib/autorouter_presets.ts | 95 ++++++++-- 6 files changed, 413 insertions(+), 47 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index 52459d69b9a..68044a45edc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -101,6 +101,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { id?: string | null; /** False for config.yaml-defined deployments, which the update and delete routes refuse. */ db_model?: boolean | null; + base_model?: string | null; created_at?: string | null; updated_at?: string | null; team_id?: string | null; @@ -122,7 +123,7 @@ export const selectAutoRouterModelGroups = (deployments: AutoRouterCandidateDepl export const selectAutoRouterDeployments = (deployments: AutoRouterDeployment[]): AutoRouterDeployment[] => deployments.filter(isAutoRouterDeployment); -const fetchAllModelDeployments = async ( +export const fetchAllModelDeployments = async ( accessToken: string, userId: string, userRole: string, @@ -151,7 +152,7 @@ const fetchAllModelDeployments = async ( * A private namespace meant an edit through ModelInfoView left this list stale, and every * future writer would have had to remember a second key. */ -const autoRouterListKey = (userId: string | null, userRole: string | null) => +export const autoRouterListKey = (userId: string | null, userRole: string | null) => modelKeys.list({ filters: { scope: "autoRouters", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx index f27c1e1c44a..f09aa1a0c84 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/AutoRoutersPanel.tsx @@ -104,6 +104,7 @@ export function AutoRoutersPanel({ accessToken, userRole, userID, teams, createS handleOk={handleCreated} accessToken={accessToken} userRole={userRole} + userId={userID} createScope={createScope} /> diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx index fd6ac13568c..57afdbbd28b 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -41,7 +41,10 @@ const optionByLabel = (label: string): HTMLElement | undefined => const isOptionDisabled = (option: HTMLElement): boolean => option.classList.contains("ant-select-item-option-disabled"); -const { mockFetchAvailableModels } = vi.hoisted(() => ({ mockFetchAvailableModels: vi.fn() })); +const { mockFetchAvailableModels, mockFetchAllModelDeployments } = vi.hoisted(() => ({ + mockFetchAvailableModels: vi.fn(), + mockFetchAllModelDeployments: vi.fn(), +})); vi.mock("../networking", () => ({ modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }), @@ -52,6 +55,11 @@ vi.mock("@/components/llm_calls/fetch_models", () => ({ fetchAvailableModels: mockFetchAvailableModels, })); +vi.mock("@/app/(dashboard)/hooks/models/useModels", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, fetchAllModelDeployments: mockFetchAllModelDeployments }; +}); + vi.mock("./handle_add_auto_router_submit", () => ({ handleAddAutoRouterSubmit: vi.fn(), })); @@ -93,6 +101,7 @@ describe("AddAutoRouterTab", () => { // test's data instead of its own mock). testQueryClient.clear(); mockFetchAvailableModels.mockResolvedValue([]); + mockFetchAllModelDeployments.mockResolvedValue([]); }); // Detailed Configuration starts collapsed so the modal opens onto just Name + Template; a caller @@ -429,6 +438,16 @@ describe("AddAutoRouterTab", () => { expect(anthropicOption.textContent).toContain("Cannot verify these models are available"); }); + it("keeps group-name presets selectable when only the deployment fetch fails", async () => { + mockFetchAvailableModels.mockResolvedValue(ALL_FAMILY_MODELS); + mockFetchAllModelDeployments.mockRejectedValue(new Error("network error")); + + renderWithProviders(); + + await waitForPresetEnabled("Anthropic Family"); + await waitForPresetEnabled("OpenAI Family"); + }); + it("disables a preset missing one of its models, naming the missing model", async () => { mockFetchAvailableModels.mockResolvedValue( ALL_FAMILY_MODELS.filter((m) => m.model_group !== ANTHROPIC_ONLY_MODEL), @@ -540,4 +559,95 @@ describe("AddAutoRouterTab", () => { expect(handleAddAutoRouterSubmit).not.toHaveBeenCalled(); }); }); + + describe("deployment-matched presets", () => { + const renamedDeploymentsFor = (presetKey: string) => + [...getRequiredModelsInPreset(getPresetByKey(presetKey)!)].map((model, index) => ({ + model_name: `renamed-${presetKey}-${index}`, + litellm_params: { model: `someprovider/${model}` }, + })); + + const groupsFor = (deployments: { model_name: string }[]): ModelGroup[] => + deployments.map((deployment) => ({ model_group: deployment.model_name, mode: "chat" })); + + const ALL_RENAMED_DEPLOYMENTS = getAllPresets().flatMap((preset) => renamedDeploymentsFor(preset.key)); + + const renamedGroupFor = (model: string): string => + ALL_RENAMED_DEPLOYMENTS.find((deployment) => deployment.litellm_params.model === `someprovider/${model}`)! + .model_name; + + it("enables a preset whose models exist only under renamed deployments, labeling the match", async () => { + mockFetchAvailableModels.mockResolvedValue(groupsFor(ALL_RENAMED_DEPLOYMENTS)); + mockFetchAllModelDeployments.mockResolvedValue(ALL_RENAMED_DEPLOYMENTS); + + renderWithProviders(); + openTemplateDropdown(); + + await waitFor(() => { + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); + }); + expect(optionByLabel("Anthropic Family")!.textContent).toContain("Matches your deployments"); + }); + + it("keeps detailed configuration open and prefills the admin's group names on apply", async () => { + const user = userEvent.setup(); + mockFetchAvailableModels.mockResolvedValue(groupsFor(ALL_RENAMED_DEPLOYMENTS)); + mockFetchAllModelDeployments.mockResolvedValue(ALL_RENAMED_DEPLOYMENTS); + + renderWithProviders(); + openTemplateDropdown(); + await waitFor(() => { + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); + }); + fireEvent.click(optionByLabel("Anthropic Family")!); + + expect(screen.getByText("Advanced: Keyword/Semantic Matching")).toBeInTheDocument(); + + await user.type(screen.getByPlaceholderText(/smart_router/i), "renamed-router"); + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + await waitFor(() => expect(handleAddAutoRouterSubmit).toHaveBeenCalled()); + expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({ + complexity_router_config: { + tiers: { + SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(renamedGroupFor), + MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(renamedGroupFor), + COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(renamedGroupFor), + REASONING: ANTHROPIC_TIERS.REASONING.map(renamedGroupFor), + }, + }, + }); + }); + + it("lists a deployment-matched preset ahead of one that stays unavailable", async () => { + const anthropicOnly = renamedDeploymentsFor("anthropic_family"); + mockFetchAvailableModels.mockResolvedValue(groupsFor(anthropicOnly)); + mockFetchAllModelDeployments.mockResolvedValue(anthropicOnly); + + renderWithProviders(); + openTemplateDropdown(); + + await waitFor(() => { + expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); + }); + const labels = Array.from(document.querySelectorAll(".ant-select-item-option")).map( + (option) => option.querySelector(".font-medium")?.textContent, + ); + expect(labels).toEqual(["Anthropic Family", "OpenAI Family", "Custom Configuration"]); + }); + + it("never lets a wildcard deployment satisfy a preset", async () => { + const wildcard = [{ model_name: "openai-wild", litellm_params: { model: "openai/*" } }]; + mockFetchAvailableModels.mockResolvedValue(groupsFor(wildcard)); + mockFetchAllModelDeployments.mockResolvedValue(wildcard); + + renderWithProviders(); + openTemplateDropdown(); + + await waitFor(() => { + expect(optionByLabel("OpenAI Family")!.textContent).toContain("Missing:"); + }); + expect(isOptionDisabled(optionByLabel("OpenAI Family")!)).toBe(true); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 45a43260bba..00d3364ac27 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -9,6 +9,7 @@ import { type ModelWriteScope } from "@/utils/modelPermissions"; import TeamDropdown from "../common_components/team_dropdown"; import { handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit"; import { fetchAvailableModels } from "@/components/llm_calls/fetch_models"; +import { autoRouterListKey, fetchAllModelDeployments } from "@/app/(dashboard)/hooks/models/useModels"; import ComplexityRouterConfig, { ComplexityRouterConfigValue, ComplexityTiers, @@ -38,6 +39,9 @@ import { getReferencedModelsError, buildEmptyPrefill, buildPresetPrefill, + buildModelAvailability, + deploymentRefsFromModelInfo, + ModelAvailability, PresetPrefill, AutoRouterPreset, } from "@/lib/autorouter_presets"; @@ -46,6 +50,7 @@ interface AddAutoRouterTabProps { handleOk: () => void; accessToken: string; userRole: string; + userId?: string | null; /** * How this caller must scope what they create. A team admin has to name a team, because * POST /model/new rejects an unscoped create from any non-proxy-admin; without the selector @@ -55,7 +60,7 @@ interface AddAutoRouterTabProps { } type PresetAvailability = - | { kind: "available" } + | { kind: "available"; viaDeployments: boolean } | { kind: "loading" } | { kind: "unverifiable" } | { kind: "missing_models"; models: readonly string[] }; @@ -110,17 +115,18 @@ const getSubmitBlockedReason = ( config: ComplexityRouterConfigValue, keywordTierRules: KeywordTierRule[], referencedModelsParams: Parameters[0], - availableModelSet: Set, + availability: ModelAvailability, ): string | null => getMissingTiersError(config.tiers) ?? getTierLabelsError(config.tier_labels) ?? getKeywordTierRulesError(keywordTierRules) ?? - getReferencedModelsError(referencedModelsParams, availableModelSet); + getReferencedModelsError(referencedModelsParams, availability); const AddAutoRouterTab: React.FC = ({ handleOk, accessToken, userRole, + userId, createScope = "unscoped-ok", }) => { const requiresTeamScope = createScope === "team-required"; @@ -163,7 +169,7 @@ const AddAutoRouterTab: React.FC = ({ const { data, - isLoading: modelsLoading, + isLoading: groupsLoading, isError: modelsError, refetch: refetchModels, } = useQuery({ @@ -171,6 +177,12 @@ const AddAutoRouterTab: React.FC = ({ queryFn: () => fetchAvailableModels(accessToken), enabled: Boolean(accessToken), }); + const { data: deployments, isLoading: deploymentsLoading } = useQuery({ + queryKey: autoRouterListKey(userId ?? "", userRole), + queryFn: () => fetchAllModelDeployments(accessToken, userId ?? "", userRole), + enabled: Boolean(accessToken), + }); + const modelsLoading = groupsLoading || deploymentsLoading; const modelInfo = React.useMemo(() => data ?? [], [data]); // react-query keeps the last successful list around when a later refetch fails, so isError alone // can't tell "never loaded" apart from "loaded, then a background refetch errored" - only the @@ -184,7 +196,22 @@ const AddAutoRouterTab: React.FC = ({ label: model_group, })); - const availableModelSet = React.useMemo(() => new Set(modelInfo.map((m) => m.model_group)), [modelInfo]); + const availability = React.useMemo( + () => + buildModelAvailability( + modelInfo.map((m) => m.model_group), + deploymentRefsFromModelInfo(deployments ?? []), + ), + [modelInfo, deployments], + ); + const groupsOnlyAvailability = React.useMemo( + () => + buildModelAvailability( + modelInfo.map((m) => m.model_group), + [], + ), + [modelInfo], + ); // A preset's models can only be trusted against a successfully loaded list. Selection and the // greyed-out state derive from this one function, so a preset that cannot be selected can never @@ -195,10 +222,22 @@ const AddAutoRouterTab: React.FC = ({ (preset: AutoRouterPreset): PresetAvailability => { if (modelsLoading) return { kind: "loading" }; if (modelsUnverifiable) return { kind: "unverifiable" }; - const missing = getMissingModelsInPreset(preset, availableModelSet); - return missing.length > 0 ? { kind: "missing_models", models: missing } : { kind: "available" }; + const missing = getMissingModelsInPreset(preset, availability); + if (missing.length > 0) return { kind: "missing_models", models: missing }; + return { + kind: "available", + viaDeployments: getMissingModelsInPreset(preset, groupsOnlyAvailability).length > 0, + }; }, - [modelsLoading, modelsUnverifiable, availableModelSet], + [modelsLoading, modelsUnverifiable, availability, groupsOnlyAvailability], + ); + + const sortedPresetOptions = React.useMemo( + () => + presets + .map((preset) => ({ preset, availability: presetAvailability(preset) })) + .sort((a, b) => Number(b.availability.kind === "available") - Number(a.availability.kind === "available")), + [presetAvailability], ); const applyPrefill = (prefill: PresetPrefill) => { @@ -222,11 +261,13 @@ const AddAutoRouterTab: React.FC = ({ const preset = getPresetByKey(presetKey); // Refuse to apply a preset whose models are not verified available. The dropdown disables // these options, so this is a guard against a stale click resolving after the list changed. - if (!preset || presetAvailability(preset).kind !== "available") return; + if (!preset) return; + const presetState = presetAvailability(preset); + if (presetState.kind !== "available") return; setSelectedPreset(presetKey); - applyPrefill(buildPresetPrefill(preset.complexity_router_config, availableModelSet)); - setDetailsExpanded(false); + applyPrefill(buildPresetPrefill(preset.complexity_router_config, availability)); + setDetailsExpanded(presetState.viaDeployments); }; const referencedModelsParams = { @@ -241,7 +282,7 @@ const AddAutoRouterTab: React.FC = ({ complexityRouterConfig, keywordTierRules, referencedModelsParams, - availableModelSet, + groupsOnlyAvailability, ); const complexityRouterConfigParams: BuildComplexityRouterConfigParams = { @@ -308,7 +349,7 @@ const AddAutoRouterTab: React.FC = ({ // same handler) fires on Enter regardless of the button's disabled state - without this check, // Enter in the name field could still create a router referencing a model that disappeared from // availableModelSet after the tiers were filled in. - const referencedModelsError = getReferencedModelsError(referencedModelsParams, availableModelSet); + const referencedModelsError = getReferencedModelsError(referencedModelsParams, groupsOnlyAvailability); if (referencedModelsError) { setShowValidationErrors(true); NotificationManager.fromBackend(referencedModelsError); @@ -405,11 +446,12 @@ const AddAutoRouterTab: React.FC = ({ optionLabelProp="label" data-testid="template-selector" > - {presets.map((preset) => { - const availability = presetAvailability(preset); - const disabledHint = presetDisabledHint(availability); + {sortedPresetOptions.map(({ preset, availability: presetState }) => { + const disabledHint = presetDisabledHint(presetState); const isDisabled = disabledHint !== null; - const hintClass = isPresetHintAlarming(availability) ? "text-red-500" : "text-gray-400"; + const hintClass = isPresetHintAlarming(presetState) ? "text-red-500" : "text-gray-400"; + const matchedHint = + presetState.kind === "available" && presetState.viaDeployments ? "Matches your deployments" : null; return ( = ({
{preset.label}
{preset.description}
{disabledHint &&
{disabledHint}
} + {matchedHint &&
{matchedHint}
} ); diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index f90341a4e31..0d965be054b 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -9,10 +9,14 @@ import { getReferencedModelsError, buildEmptyPrefill, buildPresetPrefill, + buildModelAvailability, + deploymentRefsFromModelInfo, } from "./autorouter_presets"; import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching"; import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords"; +const groupsOnly = (models: Iterable) => buildModelAvailability(models, []); + describe("autorouter_presets", () => { it("loads exactly the two model-family presets", () => { const presets = getAllPresets(); @@ -52,8 +56,8 @@ describe("autorouter_presets", () => { const required = [...getRequiredModelsInPreset(preset)]; const [held] = required; - expect(getMissingModelsInPreset(preset, new Set([held]))).toEqual(required.filter((m) => m !== held).sort()); - expect(getMissingModelsInPreset(preset, new Set(required))).toEqual([]); + expect(getMissingModelsInPreset(preset, groupsOnly([held]))).toEqual(required.filter((m) => m !== held).sort()); + expect(getMissingModelsInPreset(preset, groupsOnly(required))).toEqual([]); }); // Admins spell version numbers with either "-" or "." (claude-sonnet-4-5 vs claude-sonnet-4.5); @@ -67,14 +71,14 @@ describe("autorouter_presets", () => { const dottedSpellings = required.map((model) => model.replace(/(\d)-(\d)/g, "$1.$2")); expect(dottedSpellings).not.toEqual(required); - expect(getMissingModelsInPreset(preset, new Set(dottedSpellings))).toEqual([]); + expect(getMissingModelsInPreset(preset, groupsOnly(dottedSpellings))).toEqual([]); }); // The two-arm mirror: a differently-punctuated preset model must not be reported missing. it("does not flag a differently-punctuated model as missing via getMissingModels directly", () => { const missing = getMissingModels( { tiers: { SIMPLE: ["claude-sonnet-4-5"], MEDIUM: [], COMPLEX: [], REASONING: [] } }, - new Set(["claude-sonnet-4.5"]), + groupsOnly(["claude-sonnet-4.5"]), ); expect(missing).toEqual([]); }); @@ -90,9 +94,150 @@ describe("autorouter_presets", () => { expect(required).toEqual(new Set(["gpt-5-nano"])); }); + describe("deployment matching (underlying provider model IDs)", () => { + const availabilityFor = (modelGroup: string, underlyingModel: string) => + buildModelAvailability([modelGroup], [{ modelGroup, underlyingModels: [underlyingModel] }]); + + it.each([ + ["provider prefix", "my-claude-fast", "anthropic/claude-haiku-4-5", "claude-haiku-4-5"], + ["bedrock region+namespace+revision", "bedrock-opus", "bedrock/us.anthropic.claude-opus-5-v1:0", "claude-opus-5"], + ["bedrock date stamp", "bedrock-opus41", "bedrock/us.anthropic.claude-opus-4-1-20250805-v1:0", "claude-opus-4-1"], + ["dotted version", "team-gpt", "openai/gpt-5.4", "gpt-5.4"], + ["vertex @tag", "vertex-frontier", "vertex_ai/claude-fable-5@default", "claude-fable-5"], + ["bedrock 1m context label", "opus-1m", "bedrock/us.anthropic.claude-opus-5-v1:0[1m]", "claude-opus-5"], + ])("resolves a %s deployment and prefills the admin's group name", (_label, group, underlying, presetModel) => { + const availability = availabilityFor(group, underlying); + const config = { + tiers: { SIMPLE: [presetModel], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "heuristic" as const, + session_affinity: false, + }; + expect(getMissingModels(config, availability)).toEqual([]); + expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual([group]); + }); + + it.each([ + ["gpt-5.4", "openai/gpt-5.4-mini"], + ["gpt-5.4-mini", "openai/gpt-5.4"], + ["o3", "openai/o3-mini"], + ["o3-mini", "openai/o3"], + ["some-model", "prov/some-model-20991399"], + ])("never lets %s be satisfied by a deployment of %s", (presetModel, underlying) => { + const availability = availabilityFor("some-group", underlying); + const config = { tiers: { SIMPLE: [presetModel], MEDIUM: [], COMPLEX: [], REASONING: [] } }; + expect(getMissingModels(config, availability)).toEqual([presetModel]); + }); + + it("never indexes a wildcard deployment", () => { + const availability = availabilityFor("openai-wild", "openai/*"); + expect(availability.underlyingIndex.size).toBe(0); + }); + + it("ignores a deployment whose group is not itself an available model group", () => { + const availability = buildModelAvailability( + ["some-other-group"], + [{ modelGroup: "orphan-group", underlyingModels: ["anthropic/claude-opus-5"] }], + ); + expect(availability.underlyingIndex.size).toBe(0); + }); + + it("breaks ties between groups serving the same model deterministically, alphabetically", () => { + const availability = buildModelAvailability( + ["z-group", "a-group"], + [ + { modelGroup: "z-group", underlyingModels: ["anthropic/claude-opus-5"] }, + { modelGroup: "a-group", underlyingModels: ["bedrock/us.anthropic.claude-opus-5-v1:0"] }, + ], + ); + const config = { + tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "heuristic" as const, + session_affinity: false, + }; + expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual(["a-group"]); + }); + + it("prefers an exact group-name match over the deployment index", () => { + const availability = buildModelAvailability( + ["claude-opus-5", "renamed-opus"], + [{ modelGroup: "renamed-opus", underlyingModels: ["anthropic/claude-opus-5"] }], + ); + const config = { + tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] }, + classifier_type: "heuristic" as const, + session_affinity: false, + }; + expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual(["claude-opus-5"]); + }); + + it.each(getAllPresets().map((preset) => [preset.key, preset] as const))( + "fully resolves the %s preset through renamed deployments only", + (_key, preset) => { + const required = [...getRequiredModelsInPreset(preset)]; + const groups = required.map((_model, index) => `renamed-${index}`); + const availability = buildModelAvailability( + groups, + required.map((model, index) => ({ + modelGroup: `renamed-${index}`, + underlyingModels: [`someprovider/${model}`], + })), + ); + expect(getMissingModelsInPreset(preset, availability)).toEqual([]); + const prefilled = buildPresetPrefill(preset.complexity_router_config, availability); + const prefilledModels = Object.values(prefilled.complexityRouterConfig.tiers).flat(); + expect(prefilledModels.length).toBeGreaterThan(0); + for (const model of prefilledModels) expect(groups).toContain(model); + }, + ); + }); + + describe("deploymentRefsFromModelInfo", () => { + it("keeps litellm_params.model and model_info.base_model, drops rows with neither or no name", () => { + const refs = deploymentRefsFromModelInfo([ + { + model_name: "azure-prod", + litellm_params: { model: "azure/my-deployment" }, + model_info: { base_model: "azure/gpt-5.4" }, + }, + { model_name: "no-underlying", litellm_params: {}, model_info: {} }, + { litellm_params: { model: "openai/gpt-5.4" } }, + ]); + expect(refs).toEqual([{ modelGroup: "azure-prod", underlyingModels: ["azure/my-deployment", "azure/gpt-5.4"] }]); + }); + + it("lets an azure deployment resolve through base_model declared under litellm_params", () => { + const availability = buildModelAvailability( + ["azure-lp"], + deploymentRefsFromModelInfo([ + { + model_name: "azure-lp", + litellm_params: { model: "azure/opaque-deployment-name", base_model: "azure/gpt-5.4" }, + }, + ]), + ); + const config = { tiers: { SIMPLE: ["gpt-5.4"], MEDIUM: [], COMPLEX: [], REASONING: [] } }; + expect(getMissingModels(config, availability)).toEqual([]); + }); + + it("lets an azure deployment resolve through its admin-declared base_model", () => { + const availability = buildModelAvailability( + ["azure-prod"], + deploymentRefsFromModelInfo([ + { + model_name: "azure-prod", + litellm_params: { model: "azure/opaque-deployment-name" }, + model_info: { base_model: "azure/gpt-5.4" }, + }, + ]), + ); + const config = { tiers: { SIMPLE: ["gpt-5.4"], MEDIUM: [], COMPLEX: [], REASONING: [] } }; + expect(getMissingModels(config, availability)).toEqual([]); + }); + }); + describe("getReferencedModelsError", () => { const tiers = { SIMPLE: ["gpt-5-nano"], MEDIUM: [], COMPLEX: [], REASONING: [] }; - const available = new Set(["gpt-5-nano"]); + const available = groupsOnly(["gpt-5-nano"]); // Both fields are always populated with a model missing from `available`; only the // enabled/disabled toggles below decide whether that missing model gets reported. const params = { @@ -137,7 +282,10 @@ describe("autorouter_presets", () => { describe("buildPresetPrefill", () => { it("prefills a real bundled preset's tiers into the config", () => { const preset = getPresetByKey("anthropic_family")!; - const prefill = buildPresetPrefill(preset.complexity_router_config, getRequiredModelsInPreset(preset)); + const prefill = buildPresetPrefill( + preset.complexity_router_config, + groupsOnly(getRequiredModelsInPreset(preset)), + ); expect(prefill.complexityRouterConfig.tiers).toEqual(preset.complexity_router_config.tiers); }); @@ -153,7 +301,7 @@ describe("autorouter_presets", () => { match_threshold: 0, escalation_keywords: [], }; - const prefill = buildPresetPrefill(config, new Set(["gpt-5-nano"])); + const prefill = buildPresetPrefill(config, groupsOnly(["gpt-5-nano"])); expect(prefill.matchThreshold).toBe(0); expect(prefill.escalationKeywords).toEqual([]); }); @@ -165,7 +313,7 @@ describe("autorouter_presets", () => { classifier_type: "heuristic", session_affinity: false, }, - new Set(["gpt-5-nano"]), + groupsOnly(["gpt-5-nano"]), ); expect(prefill.matchThreshold).toBe(DEFAULT_MATCH_THRESHOLD); expect(prefill.escalationKeywords).toEqual(DEFAULT_ESCALATION_KEYWORDS); @@ -182,10 +330,10 @@ describe("autorouter_presets", () => { }; const labeled = buildPresetPrefill( { ...base, tier_labels: { SIMPLE: "Cheap", REASONING: "Deep" } }, - new Set(["gpt-5-nano"]), + groupsOnly(["gpt-5-nano"]), ); expect(labeled.complexityRouterConfig.tier_labels).toEqual({ SIMPLE: "Cheap", REASONING: "Deep" }); - expect(buildPresetPrefill(base, new Set(["gpt-5-nano"])).complexityRouterConfig.tier_labels).toBeUndefined(); + expect(buildPresetPrefill(base, groupsOnly(["gpt-5-nano"])).complexityRouterConfig.tier_labels).toBeUndefined(); }); it("rewrites a preset's model name to the caller's differently-punctuated registered spelling", () => { @@ -194,7 +342,7 @@ describe("autorouter_presets", () => { classifier_type: "heuristic" as const, session_affinity: false, }; - const prefill = buildPresetPrefill(config, new Set(["claude-sonnet-4.5"])); + const prefill = buildPresetPrefill(config, groupsOnly(["claude-sonnet-4.5"])); expect(prefill.complexityRouterConfig.tiers.SIMPLE).toEqual(["claude-sonnet-4.5"]); }); }); diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index 00b866332f1..5b6e3dc6f20 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -55,27 +55,90 @@ export const getRequiredModels = ( // only the punctuation within one version number does. const normalizeModelName = (model: string): string => model.replace(/(\d)\.(\d)/g, "$1-$2"); -// The caller's actual registered spelling for a required model, under either separator -// convention, or undefined if truly absent. Preset prefill must write THIS spelling, not the -// preset's literal string - otherwise a caller whose proxy only has the dotted form ends up with -// a tier pointing at a model name that was never registered. -const resolveAvailableModel = (requiredModel: string, availableModels: Set): string | undefined => { - if (availableModels.has(requiredModel)) return requiredModel; +export interface DeploymentModelRef { + modelGroup: string; + underlyingModels: readonly string[]; +} + +export interface ModelAvailability { + modelGroups: Set; + underlyingIndex: Map; +} + +const normalizeUnderlyingModel = (model: string): string | null => { + if (model.includes("*")) return null; + const ownName = model.slice(model.lastIndexOf("/") + 1).split("@")[0]; + const lastNamespaceSegment = normalizeModelName(ownName).split(".").at(-1) ?? ""; + const stripped = lastNamespaceSegment + .replace(/:\d+k$/i, "") + .replace(/\[\w+\]$/, "") + .replace(/-v\d+(:\d+)?$/, "") + .replace(/-20\d{2}(0[1-9]|1[0-2])(0[1-9]|[12]\d|3[01])$/, ""); + return stripped.toLowerCase() || null; +}; + +export const buildModelAvailability = ( + modelGroups: Iterable, + deployments: readonly DeploymentModelRef[], +): ModelAvailability => { + const groups = new Set(modelGroups); + const entries = deployments + .filter((deployment) => groups.has(deployment.modelGroup)) + .flatMap((deployment) => + deployment.underlyingModels + .map(normalizeUnderlyingModel) + .filter((key): key is string => key !== null) + .map((key) => ({ key, modelGroup: deployment.modelGroup })), + ); + const grouped = new Map>(); + for (const entry of entries) { + const groupsForKey = grouped.get(entry.key) ?? new Set(); + groupsForKey.add(entry.modelGroup); + grouped.set(entry.key, groupsForKey); + } + const underlyingIndex = new Map( + Array.from(grouped, ([key, groupsForKey]) => [key, Array.from(groupsForKey).sort()] as const), + ); + return { modelGroups: groups, underlyingIndex }; +}; + +export const deploymentRefsFromModelInfo = ( + rows: readonly { + model_name?: string | null; + litellm_params?: { model?: string | null; base_model?: string | null } | null; + model_info?: { base_model?: string | null } | null; + }[], +): DeploymentModelRef[] => + rows.flatMap((row) => { + const underlyingModels = [ + row.litellm_params?.model, + row.litellm_params?.base_model, + row.model_info?.base_model, + ].filter((model): model is string => Boolean(model)); + return row.model_name && underlyingModels.length > 0 ? [{ modelGroup: row.model_name, underlyingModels }] : []; + }); + +const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => { + const { modelGroups, underlyingIndex } = availability; + if (modelGroups.has(requiredModel)) return requiredModel; const normalized = normalizeModelName(requiredModel); - return Array.from(availableModels).find((available) => normalizeModelName(available) === normalized); + const groupMatch = Array.from(modelGroups).find((available) => normalizeModelName(available) === normalized); + if (groupMatch !== undefined) return groupMatch; + const key = normalizeUnderlyingModel(requiredModel); + return key === null ? undefined : underlyingIndex.get(key)?.[0]; }; export const getMissingModels = ( config: Pick, - availableModels: Set, + availability: ModelAvailability, ): string[] => - [...getRequiredModels(config)].filter((model) => resolveAvailableModel(model, availableModels) === undefined).sort(); + [...getRequiredModels(config)].filter((model) => resolveAvailableModel(model, availability) === undefined).sort(); export const getRequiredModelsInPreset = (preset: AutoRouterPreset): Set => getRequiredModels(preset.complexity_router_config); -export const getMissingModelsInPreset = (preset: AutoRouterPreset, availableModels: Set): string[] => - getMissingModels(preset.complexity_router_config, availableModels); +export const getMissingModelsInPreset = (preset: AutoRouterPreset, availability: ModelAvailability): string[] => + getMissingModels(preset.complexity_router_config, availability); // Checks the config actually being built (whether it arrived via a preset prefill or was typed by // hand - the two are indistinguishable once the caller has started editing), not a preset's @@ -91,7 +154,7 @@ export const getReferencedModelsError = ( semanticMatchingEnabled: boolean; embeddingModel: string | undefined; }, - availableModels: Set, + availability: ModelAvailability, ): string | null => { const missing = getMissingModels( { @@ -99,7 +162,7 @@ export const getReferencedModelsError = ( classifier_llm_config: params.classifierType === "llm" ? params.classifierLlmConfig : undefined, embedding_model: params.semanticMatchingEnabled ? params.embeddingModel : undefined, }, - availableModels, + availability, ); return missing.length > 0 ? `Model(s) no longer available: ${missing.join(", ")}` : null; }; @@ -133,16 +196,16 @@ export const buildEmptyPrefill = (): PresetPrefill => ({ // `??`, never `||`: a preset's match_threshold: 0 or escalation_keywords: [] is a deliberate, // falsy value that must survive the prefill, not get silently replaced by the default. // -// `availableModels` is required, not optional: every model reference gets rewritten to the +// `availability` is required, not optional: every model reference gets rewritten to the // caller's actual registered spelling (resolveAvailableModel), which may differ from the preset's // literal string by version-separator punctuation alone. Called only after presetAvailability has // already confirmed every required model resolves, so falling back to the preset's own string // when a model somehow doesn't resolve is unreachable in practice, not a silent-failure path. export const buildPresetPrefill = ( config: ComplexityRouterConfigPayload, - availableModels: Set, + availability: ModelAvailability, ): PresetPrefill => { - const resolve = (model: string): string => resolveAvailableModel(model, availableModels) ?? model; + const resolve = (model: string): string => resolveAvailableModel(model, availability) ?? model; const resolveTier = (models: string[]): string[] => models.map(resolve); return {