From 0c1c3e18d5250ec3a0e1e3f287e0b93e2906d900 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Wed, 23 Sep 2026 17:33:22 -0700 Subject: [PATCH] fix(ui): prefer native providers in auto-router presets (#42639) --- .../app/(dashboard)/hooks/models/useModels.ts | 1 + .../add_auto_router_tab.integration.test.tsx | 32 ++- .../components/add_model/auto_setup.test.ts | 38 +++- .../src/lib/autorouter_presets.test.ts | 209 ++++++++++++++++-- .../src/lib/autorouter_presets.ts | 87 ++++++-- 5 files changed, 322 insertions(+), 45 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index 579ee7ff81a..b5cc329d4c9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -99,6 +99,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { litellm_params?: { model?: string | null; base_model?: string | null; + custom_llm_provider?: string | null; complexity_router_config?: unknown; complexity_router_default_model?: string | null; auto_router_config?: unknown; diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx index 2779eec752f..0337d35d698 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.integration.test.tsx @@ -1660,10 +1660,6 @@ describe("AddAutoRouterTab", () => { const ALL_RENAMED_DEPLOYMENTS = getAllPresets().flatMap((preset) => renamedDeploymentsFor(preset.key)); - const renamedGroupFor = (model: string): string => - ALL_RENAMED_DEPLOYMENTS.find((deployment) => deployment.litellm_params.model === `someprovider/${model}`)! - .model_name; - it("enables a preset whose models exist only under renamed deployments, labeling the match", async () => { mockFetchAvailableModels.mockResolvedValue(groupsFor(ALL_RENAMED_DEPLOYMENTS)); mockFetchAllModelDeployments.mockResolvedValue(ALL_RENAMED_DEPLOYMENTS); @@ -1677,10 +1673,23 @@ describe("AddAutoRouterTab", () => { expect(optionByLabel("Anthropic Family")!).toHaveTextContent(/Matches your deployments/); }); - it("keeps detailed configuration open and prefills the admin's group names on apply", async () => { + it("keeps detailed configuration open and submits native group names when cloud twins are available", async () => { const user = userEvent.setup(); - mockFetchAvailableModels.mockResolvedValue(groupsFor(ALL_RENAMED_DEPLOYMENTS)); - mockFetchAllModelDeployments.mockResolvedValue(ALL_RENAMED_DEPLOYMENTS); + const nativeDeployments = renamedDeploymentsFor("anthropic_family").map((deployment) => ({ + ...deployment, + litellm_params: { model: deployment.litellm_params.model.replace("someprovider/", "anthropic/") }, + })); + const nativeGroupFor = (model: string): string => + nativeDeployments.find((deployment) => deployment.litellm_params.model === `anthropic/${model}`)!.model_name; + const cloudDeployments = nativeDeployments.map((deployment) => ({ + model_name: `a-cloud-${deployment.model_name}`, + litellm_params: { + model: `bedrock/us.anthropic.${deployment.litellm_params.model.split("/")[1]}-v1:0`, + }, + })); + const deployments = [...cloudDeployments, ...nativeDeployments]; + mockFetchAvailableModels.mockResolvedValue(groupsFor(deployments)); + mockFetchAllModelDeployments.mockResolvedValue(deployments); renderWithProviders(); openTemplateDropdown(); @@ -1688,6 +1697,7 @@ describe("AddAutoRouterTab", () => { expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false); }); await selectTemplate("Anthropic Family"); + expectTierModel("Complex", nativeGroupFor(ANTHROPIC_TIERS.COMPLEX[0])); openAutoRouterAdvanced("Keyword/Semantic Matching"); @@ -1701,10 +1711,10 @@ describe("AddAutoRouterTab", () => { expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({ complexity_router_config: { tiers: { - SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(renamedGroupFor), - MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(renamedGroupFor), - COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(renamedGroupFor), - REASONING: ANTHROPIC_TIERS.REASONING.map(renamedGroupFor), + SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(nativeGroupFor), + MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(nativeGroupFor), + COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(nativeGroupFor), + REASONING: ANTHROPIC_TIERS.REASONING.map(nativeGroupFor), }, }, }); diff --git a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts index c5784db4501..a6e81bbc184 100644 --- a/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels"; -import { buildModelAvailability } from "@/lib/autorouter_presets"; +import { buildModelAvailability, deploymentRefsFromModelInfo } from "@/lib/autorouter_presets"; import { buildAutomaticRouterConfig, buildPreferredTierModels, type PreferredTierModels } from "./auto_setup"; const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" })); @@ -54,6 +54,42 @@ describe("buildPreferredTierModels", () => { }); describe("buildAutomaticRouterConfig", () => { + it("selects native Terra and Sol groups with their reasoning settings, retaining cloud fallback", () => { + const modelNames = ["gpt-5.6-terra", "gpt-5.6-sol"]; + const deployments = modelNames.flatMap((model) => [ + deployment(model, `azure/${model}`), + deployment(`z-native-${model}`, `openai/${model}`), + ]); + const available = deployments.map(({ model_name }) => reasoningModel(model_name!, ["none", "high"])); + const availability = buildModelAvailability( + available.map(({ model_group }) => model_group), + deploymentRefsFromModelInfo(deployments), + ); + const preferred = buildPreferredTierModels([], availability); + const config = buildAutomaticRouterConfig(available, deployments, preferred); + + expect(tierModels(config)).toEqual([ + "z-native-gpt-5.6-terra", + "z-native-gpt-5.6-terra", + "z-native-gpt-5.6-sol", + "z-native-gpt-5.6-sol", + ]); + expect(config?.tier_model_params).toEqual({ + REASONING: { "z-native-gpt-5.6-sol": { reasoning_effort: "high" } }, + }); + + const cloudOnly = models(...modelNames); + const cloudAvailability = buildModelAvailability(modelNames, deploymentRefsFromModelInfo(deployments)); + const cloudPreferred = buildPreferredTierModels([], cloudAvailability); + + expect(tierModels(buildAutomaticRouterConfig(cloudOnly, deployments, cloudPreferred))).toEqual([ + "gpt-5.6-terra", + "gpt-5.6-terra", + "gpt-5.6-sol", + "gpt-5.6-sol", + ]); + }); + it("selects one preferred model for each tier", () => { const preferred: PreferredTierModels = { SIMPLE: ["simple"], diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts index 1d75a1dc7d2..c67befe8fa5 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts @@ -13,6 +13,7 @@ import { buildModelAvailability, deploymentRefsFromModelInfo, normalizeModelName, + resolveAvailableModel, resolveAvailableModels, } from "./autorouter_presets"; import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching"; @@ -393,28 +394,140 @@ describe("autorouter_presets", () => { expect(resolveAvailableModels("anthropic/claude-sonnet-5", availability)).toEqual(["a-group", "z-group"]); }); - it("breaks ties between groups serving the same model deterministically, alphabetically", () => { + it.each([ + ["OpenAI", getPresetByKey("openai_family")!.complexity_router_config.tiers.MEDIUM[0], "openai", "azure"], + [ + "Anthropic", + getPresetByKey("anthropic_family")!.complexity_router_config.tiers.COMPLEX[0], + "anthropic", + "bedrock", + ], + ["Gemini", getPresetByKey("gemini_family")!.complexity_router_config.tiers.SIMPLE[0], "gemini", "vertex_ai"], + ["DeepSeek", getPresetByKey("lite")!.complexity_router_config.tiers.SIMPLE[0], "deepseek", "openrouter"], + ["Muse", getPresetByKey("lite")!.complexity_router_config.tiers.MEDIUM[0], "meta", "openrouter"], + ["Kimi", getPresetByKey("lite")!.complexity_router_config.tiers.COMPLEX[0], "moonshot", "openrouter"], + ["Grok", "grok-4.7", "xai", "openrouter"], + ])( + "prefills %s through its native provider and falls back when only the cloud group is available", + (_family, model, native, cloud) => { + const deployments = [ + { modelGroup: "a-cloud", underlyingModels: [`${cloud}/${model}`] }, + { modelGroup: "z-native", underlyingModels: [`${native}/${model}`] }, + ]; + const config = { + tiers: { SIMPLE: [model], MEDIUM: [], COMPLEX: [], REASONING: [] }, + tier_model_configs: { SIMPLE: [{ model_name: model, litellm_params: { reasoning_effort: "high" } }] }, + classifier_type: "llm" as const, + classifier_llm_config: { model, timeout_ms: 3000 }, + classification_mode: "every_request" as const, + session_affinity: false, + deployment_affinity: true, + modality_routing: false, + modality_pin_override: false, + }; + + for (const [groups, selected] of [ + [["a-cloud", "z-native"], "z-native"], + [["a-cloud"], "a-cloud"], + ] as const) { + const availability = buildModelAvailability(groups, deployments); + const prefill = buildPresetPrefill(config, availability).complexityRouterConfig; + + expect(prefill.tiers.SIMPLE).toEqual([selected]); + expect(prefill.tier_model_params).toEqual({ SIMPLE: { [selected]: { reasoning_effort: "high" } } }); + expect(prefill.classifier_llm_config).toEqual({ model: selected, timeout_ms: 3000 }); + } + }, + ); + + it.each(["claude-opus-5-5", "claude-opus-5.5"])( + "prefers a native deployment over the cloud group named %s", + (cloudGroup) => { + const availability = buildModelAvailability( + [cloudGroup, "z-native"], + [ + { modelGroup: cloudGroup, underlyingModels: ["bedrock/us.anthropic.claude-opus-5-5-v1:0"] }, + { modelGroup: "z-native", underlyingModels: ["anthropic/claude-opus-5-5"] }, + ], + ); + + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("z-native"); + expect(resolveAvailableModels("claude-opus-5-5", availability)).toEqual([cloudGroup]); + }, + ); + + it("breaks ties between native groups alphabetically regardless of deployment order", () => { const availability = buildModelAvailability( - ["z-group", "a-group"], + ["z-native", "a-native"], [ - { modelGroup: "z-group", underlyingModels: ["anthropic/claude-opus-5"] }, - { modelGroup: "a-group", underlyingModels: ["bedrock/us.anthropic.claude-opus-5-v1:0"] }, + { modelGroup: "z-native", underlyingModels: ["anthropic/claude-opus-5-5"] }, + { modelGroup: "a-native", underlyingModels: ["anthropic/claude-opus-5-5"] }, ], ); - const config = { - tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] }, - classifier_type: "heuristic" as const, - classification_mode: "every_request" as const, - session_affinity: false, - deployment_affinity: true, - }; - expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual(["a-group"]); + + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("a-native"); }); - it("prefers an exact group-name match over the deployment index", () => { + it.each(["gpt-6-sol", "claude-opus-5-5"])("recognizes the native default of bare %s", (model) => { + const availability = buildModelAvailability( + ["a-cloud", "z-native"], + [ + { modelGroup: "a-cloud", underlyingModels: [`openrouter/${model}`] }, + { modelGroup: "z-native", underlyingModels: [model] }, + ], + ); + + expect(resolveAvailableModel(model, availability)).toBe("z-native"); + }); + + it.each(["bedrock/claude-opus-5-5", "unknown-model"])( + "prefers an exclusively native group over one that also routes to %s", + (otherModel) => { + const deployments = [ + { modelGroup: "a-cloud", underlyingModels: ["bedrock/claude-opus-5-5"] }, + { modelGroup: "b-mixed", underlyingModels: ["anthropic/claude-opus-5-5"] }, + { modelGroup: "b-mixed", underlyingModels: [otherModel] }, + { modelGroup: "z-native", underlyingModels: ["anthropic/claude-opus-5-5"] }, + ]; + const availability = buildModelAvailability(["a-cloud", "b-mixed", "z-native"], deployments); + + expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("z-native"); + const noNativeGroup = buildModelAvailability(["a-cloud", "b-mixed"], deployments); + expect(resolveAvailableModel("claude-opus-5-5", noNativeGroup)).toBe("a-cloud"); + }, + ); + + it.each([ + { model: "azure/opaque-deployment", base_model: "openai/gpt-6-sol" }, + { model: "openai/gpt-6-sol", custom_llm_provider: "openrouter" }, + ])("keeps cloud routing authoritative over native-looking model metadata: %j", (litellmParams) => { + const availability = buildModelAvailability( + ["a-cloud", "z-native"], + deploymentRefsFromModelInfo([ + { model_name: "a-cloud", litellm_params: litellmParams, model_info: { base_model: "openai/gpt-6-sol" } }, + { model_name: "z-native", litellm_params: { model: "openai/gpt-6-sol" } }, + ]), + ); + + expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("z-native"); + }); + + it("recognizes an explicit native provider on an otherwise unqualified model", () => { + const availability = buildModelAvailability( + ["a-cloud", "z-native"], + deploymentRefsFromModelInfo([ + { model_name: "a-cloud", litellm_params: { model: "openrouter/meta/muse-spark-1.3" } }, + { model_name: "z-native", litellm_params: { model: "muse-spark-1.3", custom_llm_provider: "meta" } }, + ]), + ); + + expect(resolveAvailableModel("muse-spark-1.3", availability)).toBe("z-native"); + }); + + it("preserves exact group-name precedence when no known native deployment is available", () => { const availability = buildModelAvailability( ["claude-opus-5", "renamed-opus"], - [{ modelGroup: "renamed-opus", underlyingModels: ["anthropic/claude-opus-5"] }], + [{ modelGroup: "renamed-opus", underlyingModels: ["bedrock/us.anthropic.claude-opus-5-v1:0"] }], ); const config = { tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] }, @@ -573,6 +686,70 @@ describe("autorouter_presets", () => { ]); }); + it.each(["native/*", "*"])("ranks wildcard groups using their routing deployment: %s", (nativePattern) => { + const nativeGroup = nativePattern === "*" ? "openai/gpt-6-sol" : "native/gpt-6-sol"; + const availability = buildModelAvailability( + ["azure/gpt-6-sol", nativeGroup], + [ + { modelGroup: "azure/*", underlyingModels: ["openrouter/*"] }, + { modelGroup: nativePattern, underlyingModels: ["openai/*"] }, + ], + ); + + expect(resolveAvailableModel("gpt-6-sol", availability)).toBe(nativeGroup); + }); + + it("does not treat a native-looking wildcard group as native when its deployment uses the cloud", () => { + const availability = buildModelAvailability( + ["openai/gpt-6-sol", "z-native/gpt-6-sol"], + [ + { modelGroup: "openai/*", underlyingModels: ["azure/*"] }, + { modelGroup: "z-native/*", underlyingModels: ["openai/*"] }, + ], + ); + + expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("z-native/gpt-6-sol"); + }); + + it("keeps literal native deployments ahead of a matching cloud wildcard", () => { + const availability = buildModelAvailability( + ["a-cloud", "team/gpt-6-sol"], + [ + { modelGroup: "a-cloud", underlyingModels: ["azure/gpt-6-sol"] }, + { modelGroup: "team/gpt-6-sol", underlyingModels: ["openai/gpt-6-sol"] }, + { modelGroup: "team/*", underlyingModels: ["azure/*"] }, + ], + ); + + expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("team/gpt-6-sol"); + }); + + it("does not promote a bare-star expansion when its routing group also contains a cloud deployment", () => { + const availability = buildModelAvailability( + ["openai/gpt-6-sol", "z-native"], + [ + { modelGroup: "*", underlyingModels: ["openai/*"] }, + { modelGroup: "*", underlyingModels: ["azure/*"] }, + { modelGroup: "z-native", underlyingModels: ["openai/gpt-6-sol"] }, + ], + ); + + expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("z-native"); + }); + + it("retains fallback ordering when overlapping wildcard routes have different providers", () => { + const availability = buildModelAvailability( + ["a-cloud", "team/gpt-6-sol"], + [ + { modelGroup: "a-cloud", underlyingModels: ["azure/gpt-6-sol"] }, + { modelGroup: "team/*", underlyingModels: ["azure/*"] }, + { modelGroup: "team/gpt-*", underlyingModels: ["openai/gpt-*"] }, + ], + ); + + expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("a-cloud"); + }); + it.each(getAllPresets().map((preset) => [preset.key, preset] as const))( "fully resolves the %s preset through wildcard-expanded groups only", (_key, preset) => { @@ -602,7 +779,9 @@ describe("autorouter_presets", () => { { model_name: "no-underlying", litellm_params: {}, model_info: {} }, { litellm_params: { model: "openai/gpt-5.4" } }, ]); - expect(refs).toEqual([{ modelGroup: "azure-prod", underlyingModels: ["azure/my-deployment", "azure/gpt-5.4"] }]); + expect(refs).toEqual([ + { modelGroup: "azure-prod", underlyingModels: ["azure/my-deployment", "azure/gpt-5.4"], provider: "azure" }, + ]); }); it("lets an azure deployment resolve through base_model declared under litellm_params", () => { diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts index 8cf461d77b9..bbbf151e49b 100644 --- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts +++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts @@ -65,13 +65,37 @@ export const normalizeModelName = (model: string): string => model.replace(/(\d) export interface DeploymentModelRef { modelGroup: string; underlyingModels: readonly string[]; + provider?: string; } export interface ModelAvailability { modelGroups: Set; underlyingIndex: Map; + nativeUnderlyingIndex: Map; } +const NATIVE_MODEL_PROVIDERS: readonly (readonly [RegExp, string])[] = [ + [/^(gpt-|o\d|text-embedding-)/, "openai"], + [/^claude-/, "anthropic"], + [/^gemini-/, "gemini"], + [/^deepseek-/, "deepseek"], + [/^muse-/, "meta"], + [/^kimi-/, "moonshot"], + [/^grok-/, "xai"], +]; + +const nativeModelProvider = (model: string): string | undefined => + NATIVE_MODEL_PROVIDERS.find(([pattern]) => pattern.test(model))?.[1]; + +const routingProvider = (model: string): string => { + if (model.includes("/")) return model.split("/")[0]; + const native = nativeModelProvider(model); + return native === "openai" || native === "anthropic" ? native : ""; +}; + +const deploymentProvider = (deployment: DeploymentModelRef): string => + deployment.provider ?? routingProvider(deployment.underlyingModels[0] ?? ""); + const normalizeUnderlyingModel = (model: string): string | null => { if (model.includes("*")) return null; const ownName = model.slice(model.lastIndexOf("/") + 1).split("@")[0]; @@ -107,32 +131,42 @@ export const buildModelAvailability = ( deployments: readonly DeploymentModelRef[], ): ModelAvailability => { const groups = new Set(modelGroups); + const deploymentGroups = new Set(deployments.map((deployment) => deployment.modelGroup)); + const deploymentProviders = new Map>(); + for (const deployment of deployments) { + const providers = deploymentProviders.get(deployment.modelGroup) ?? new Set(); + providers.add(deploymentProvider(deployment)); + deploymentProviders.set(deployment.modelGroup, providers); + } const literalEntries = deployments .filter((deployment) => groups.has(deployment.modelGroup)) .flatMap((deployment) => deployment.underlyingModels .map(normalizeUnderlyingModel) - .filter((key): key is string => key !== null) - .map((key) => ({ key, modelGroup: deployment.modelGroup })), + .map((key) => ({ key, modelGroup: deployment.modelGroup, sourceGroup: deployment.modelGroup })), ); // Mirrors get_known_models_from_wildcard: a bare "*" model_name expands via its underlying // wildcard (or not at all), and a wildcard without a "/" expands to nothing. - const wildcardPatterns = Array.from( - new Set( - deployments - .flatMap((deployment) => - deployment.modelGroup === "*" ? deployment.underlyingModels : [deployment.modelGroup], - ) - .filter((pattern) => pattern !== "*" && pattern.includes("*") && pattern.includes("/")), - ), + const wildcardPatterns = deployments.flatMap((deployment) => + (deployment.modelGroup === "*" ? deployment.underlyingModels : [deployment.modelGroup]) + .filter((pattern) => pattern !== "*" && pattern.includes("*") && pattern.includes("/")) + .map((pattern) => ({ pattern, sourceGroup: deployment.modelGroup })), ); const wildcardEntries = Array.from(groups) - .filter((group) => !group.includes("*") && wildcardPatterns.some((pattern) => matchesWildcard(pattern, group))) - .map((group) => ({ key: normalizeUnderlyingModel(group), modelGroup: group })) - .filter((entry): entry is { key: string; modelGroup: string } => entry.key !== null); + .filter((group) => !group.includes("*") && !deploymentGroups.has(group)) + .flatMap((group) => + wildcardPatterns + .filter(({ pattern }) => matchesWildcard(pattern, group)) + .map(({ sourceGroup }) => ({ key: normalizeUnderlyingModel(group), modelGroup: group, sourceGroup })), + ); const entries = [...literalEntries, ...wildcardEntries]; const grouped = new Map>(); + const providersByGroup = new Map>(); for (const entry of entries) { + const providers = providersByGroup.get(entry.modelGroup) ?? new Set(); + for (const provider of deploymentProviders.get(entry.sourceGroup) ?? []) providers.add(provider); + providersByGroup.set(entry.modelGroup, providers); + if (entry.key === null) continue; const groupsForKey = grouped.get(entry.key) ?? new Set(); groupsForKey.add(entry.modelGroup); grouped.set(entry.key, groupsForKey); @@ -140,13 +174,23 @@ export const buildModelAvailability = ( const underlyingIndex = new Map( Array.from(grouped, ([key, groupsForKey]) => [key, Array.from(groupsForKey).sort()] as const), ); - return { modelGroups: groups, underlyingIndex }; + const nativeUnderlyingIndex = new Map( + Array.from(underlyingIndex, ([key, matches]) => [ + key, + matches.filter((group) => { + const native = nativeModelProvider(key); + const providers = providersByGroup.get(group); + return native !== undefined && providers?.size === 1 && providers.has(native); + }), + ]), + ); + return { modelGroups: groups, underlyingIndex, nativeUnderlyingIndex }; }; export const deploymentRefsFromModelInfo = ( rows: readonly { model_name?: string | null; - litellm_params?: { model?: string | null; base_model?: string | null } | null; + litellm_params?: { model?: string | null; base_model?: string | null; custom_llm_provider?: string | null } | null; model_info?: { base_model?: string | null } | null; }[], ): DeploymentModelRef[] => @@ -156,7 +200,10 @@ export const deploymentRefsFromModelInfo = ( row.litellm_params?.base_model, row.model_info?.base_model, ].filter((model): model is string => Boolean(model)); - return row.model_name && underlyingModels.length > 0 ? [{ modelGroup: row.model_name, underlyingModels }] : []; + const provider = row.litellm_params?.custom_llm_provider || routingProvider(row.litellm_params?.model ?? ""); + return row.model_name && underlyingModels.length > 0 + ? [{ modelGroup: row.model_name, underlyingModels, provider }] + : []; }); export const resolveAvailableModels = (requiredModel: string, availability: ModelAvailability): readonly string[] => { @@ -169,8 +216,12 @@ export const resolveAvailableModels = (requiredModel: string, availability: Mode return key === null ? [] : underlyingIndex.get(key) ?? []; }; -export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => - resolveAvailableModels(requiredModel, availability)[0]; +export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => { + const key = normalizeUnderlyingModel(requiredModel); + const nativeMatches = key === null ? [] : availability.nativeUnderlyingIndex.get(key) ?? []; + const matches = resolveAvailableModels(requiredModel, availability); + return matches.find((model) => nativeMatches.includes(model)) ?? nativeMatches[0] ?? matches[0]; +}; export const getMissingModels = ( config: Parameters[0],