mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
fix(ui): prefer native providers in auto-router presets (#42639)
This commit is contained in:
tin-berri
GitHub
parent
bab86555e7
commit
0c1c3e18d5
5 changed files with 322 additions and 45 deletions
|
|
@ -99,6 +99,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment {
|
|||
litellm_params?: {
|
||||
model?: string | null;
|
||||
base_model?: string | null;
|
||||
custom_llm_provider?: string | null;
|
||||
complexity_router_config?: unknown;
|
||||
complexity_router_default_model?: string | null;
|
||||
auto_router_config?: unknown;
|
||||
|
|
|
|||
|
|
@ -1660,10 +1660,6 @@ describe("AddAutoRouterTab", () => {
|
|||
|
||||
const ALL_RENAMED_DEPLOYMENTS = getAllPresets().flatMap((preset) => renamedDeploymentsFor(preset.key));
|
||||
|
||||
const renamedGroupFor = (model: string): string =>
|
||||
ALL_RENAMED_DEPLOYMENTS.find((deployment) => deployment.litellm_params.model === `someprovider/${model}`)!
|
||||
.model_name;
|
||||
|
||||
it("enables a preset whose models exist only under renamed deployments, labeling the match", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue(groupsFor(ALL_RENAMED_DEPLOYMENTS));
|
||||
mockFetchAllModelDeployments.mockResolvedValue(ALL_RENAMED_DEPLOYMENTS);
|
||||
|
|
@ -1677,10 +1673,23 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(optionByLabel("Anthropic Family")!).toHaveTextContent(/Matches your deployments/);
|
||||
});
|
||||
|
||||
it("keeps detailed configuration open and prefills the admin's group names on apply", async () => {
|
||||
it("keeps detailed configuration open and submits native group names when cloud twins are available", async () => {
|
||||
const user = userEvent.setup();
|
||||
mockFetchAvailableModels.mockResolvedValue(groupsFor(ALL_RENAMED_DEPLOYMENTS));
|
||||
mockFetchAllModelDeployments.mockResolvedValue(ALL_RENAMED_DEPLOYMENTS);
|
||||
const nativeDeployments = renamedDeploymentsFor("anthropic_family").map((deployment) => ({
|
||||
...deployment,
|
||||
litellm_params: { model: deployment.litellm_params.model.replace("someprovider/", "anthropic/") },
|
||||
}));
|
||||
const nativeGroupFor = (model: string): string =>
|
||||
nativeDeployments.find((deployment) => deployment.litellm_params.model === `anthropic/${model}`)!.model_name;
|
||||
const cloudDeployments = nativeDeployments.map((deployment) => ({
|
||||
model_name: `a-cloud-${deployment.model_name}`,
|
||||
litellm_params: {
|
||||
model: `bedrock/us.anthropic.${deployment.litellm_params.model.split("/")[1]}-v1:0`,
|
||||
},
|
||||
}));
|
||||
const deployments = [...cloudDeployments, ...nativeDeployments];
|
||||
mockFetchAvailableModels.mockResolvedValue(groupsFor(deployments));
|
||||
mockFetchAllModelDeployments.mockResolvedValue(deployments);
|
||||
|
||||
renderWithProviders(<Harness />);
|
||||
openTemplateDropdown();
|
||||
|
|
@ -1688,6 +1697,7 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(isOptionDisabled(optionByLabel("Anthropic Family")!)).toBe(false);
|
||||
});
|
||||
await selectTemplate("Anthropic Family");
|
||||
expectTierModel("Complex", nativeGroupFor(ANTHROPIC_TIERS.COMPLEX[0]));
|
||||
|
||||
openAutoRouterAdvanced("Keyword/Semantic Matching");
|
||||
|
||||
|
|
@ -1701,10 +1711,10 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(vi.mocked(handleAddAutoRouterSubmit).mock.calls.at(-1)?.[0]).toMatchObject({
|
||||
complexity_router_config: {
|
||||
tiers: {
|
||||
SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(renamedGroupFor),
|
||||
MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(renamedGroupFor),
|
||||
COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(renamedGroupFor),
|
||||
REASONING: ANTHROPIC_TIERS.REASONING.map(renamedGroupFor),
|
||||
SIMPLE: ANTHROPIC_TIERS.SIMPLE.map(nativeGroupFor),
|
||||
MEDIUM: ANTHROPIC_TIERS.MEDIUM.map(nativeGroupFor),
|
||||
COMPLEX: ANTHROPIC_TIERS.COMPLEX.map(nativeGroupFor),
|
||||
REASONING: ANTHROPIC_TIERS.REASONING.map(nativeGroupFor),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import { buildModelAvailability } from "@/lib/autorouter_presets";
|
||||
import { buildModelAvailability, deploymentRefsFromModelInfo } from "@/lib/autorouter_presets";
|
||||
import { buildAutomaticRouterConfig, buildPreferredTierModels, type PreferredTierModels } from "./auto_setup";
|
||||
|
||||
const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" }));
|
||||
|
|
@ -54,6 +54,42 @@ describe("buildPreferredTierModels", () => {
|
|||
});
|
||||
|
||||
describe("buildAutomaticRouterConfig", () => {
|
||||
it("selects native Terra and Sol groups with their reasoning settings, retaining cloud fallback", () => {
|
||||
const modelNames = ["gpt-5.6-terra", "gpt-5.6-sol"];
|
||||
const deployments = modelNames.flatMap((model) => [
|
||||
deployment(model, `azure/${model}`),
|
||||
deployment(`z-native-${model}`, `openai/${model}`),
|
||||
]);
|
||||
const available = deployments.map(({ model_name }) => reasoningModel(model_name!, ["none", "high"]));
|
||||
const availability = buildModelAvailability(
|
||||
available.map(({ model_group }) => model_group),
|
||||
deploymentRefsFromModelInfo(deployments),
|
||||
);
|
||||
const preferred = buildPreferredTierModels([], availability);
|
||||
const config = buildAutomaticRouterConfig(available, deployments, preferred);
|
||||
|
||||
expect(tierModels(config)).toEqual([
|
||||
"z-native-gpt-5.6-terra",
|
||||
"z-native-gpt-5.6-terra",
|
||||
"z-native-gpt-5.6-sol",
|
||||
"z-native-gpt-5.6-sol",
|
||||
]);
|
||||
expect(config?.tier_model_params).toEqual({
|
||||
REASONING: { "z-native-gpt-5.6-sol": { reasoning_effort: "high" } },
|
||||
});
|
||||
|
||||
const cloudOnly = models(...modelNames);
|
||||
const cloudAvailability = buildModelAvailability(modelNames, deploymentRefsFromModelInfo(deployments));
|
||||
const cloudPreferred = buildPreferredTierModels([], cloudAvailability);
|
||||
|
||||
expect(tierModels(buildAutomaticRouterConfig(cloudOnly, deployments, cloudPreferred))).toEqual([
|
||||
"gpt-5.6-terra",
|
||||
"gpt-5.6-terra",
|
||||
"gpt-5.6-sol",
|
||||
"gpt-5.6-sol",
|
||||
]);
|
||||
});
|
||||
|
||||
it("selects one preferred model for each tier", () => {
|
||||
const preferred: PreferredTierModels = {
|
||||
SIMPLE: ["simple"],
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
buildModelAvailability,
|
||||
deploymentRefsFromModelInfo,
|
||||
normalizeModelName,
|
||||
resolveAvailableModel,
|
||||
resolveAvailableModels,
|
||||
} from "./autorouter_presets";
|
||||
import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching";
|
||||
|
|
@ -393,28 +394,140 @@ describe("autorouter_presets", () => {
|
|||
expect(resolveAvailableModels("anthropic/claude-sonnet-5", availability)).toEqual(["a-group", "z-group"]);
|
||||
});
|
||||
|
||||
it("breaks ties between groups serving the same model deterministically, alphabetically", () => {
|
||||
it.each([
|
||||
["OpenAI", getPresetByKey("openai_family")!.complexity_router_config.tiers.MEDIUM[0], "openai", "azure"],
|
||||
[
|
||||
"Anthropic",
|
||||
getPresetByKey("anthropic_family")!.complexity_router_config.tiers.COMPLEX[0],
|
||||
"anthropic",
|
||||
"bedrock",
|
||||
],
|
||||
["Gemini", getPresetByKey("gemini_family")!.complexity_router_config.tiers.SIMPLE[0], "gemini", "vertex_ai"],
|
||||
["DeepSeek", getPresetByKey("lite")!.complexity_router_config.tiers.SIMPLE[0], "deepseek", "openrouter"],
|
||||
["Muse", getPresetByKey("lite")!.complexity_router_config.tiers.MEDIUM[0], "meta", "openrouter"],
|
||||
["Kimi", getPresetByKey("lite")!.complexity_router_config.tiers.COMPLEX[0], "moonshot", "openrouter"],
|
||||
["Grok", "grok-4.7", "xai", "openrouter"],
|
||||
])(
|
||||
"prefills %s through its native provider and falls back when only the cloud group is available",
|
||||
(_family, model, native, cloud) => {
|
||||
const deployments = [
|
||||
{ modelGroup: "a-cloud", underlyingModels: [`${cloud}/${model}`] },
|
||||
{ modelGroup: "z-native", underlyingModels: [`${native}/${model}`] },
|
||||
];
|
||||
const config = {
|
||||
tiers: { SIMPLE: [model], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
tier_model_configs: { SIMPLE: [{ model_name: model, litellm_params: { reasoning_effort: "high" } }] },
|
||||
classifier_type: "llm" as const,
|
||||
classifier_llm_config: { model, timeout_ms: 3000 },
|
||||
classification_mode: "every_request" as const,
|
||||
session_affinity: false,
|
||||
deployment_affinity: true,
|
||||
modality_routing: false,
|
||||
modality_pin_override: false,
|
||||
};
|
||||
|
||||
for (const [groups, selected] of [
|
||||
[["a-cloud", "z-native"], "z-native"],
|
||||
[["a-cloud"], "a-cloud"],
|
||||
] as const) {
|
||||
const availability = buildModelAvailability(groups, deployments);
|
||||
const prefill = buildPresetPrefill(config, availability).complexityRouterConfig;
|
||||
|
||||
expect(prefill.tiers.SIMPLE).toEqual([selected]);
|
||||
expect(prefill.tier_model_params).toEqual({ SIMPLE: { [selected]: { reasoning_effort: "high" } } });
|
||||
expect(prefill.classifier_llm_config).toEqual({ model: selected, timeout_ms: 3000 });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["claude-opus-5-5", "claude-opus-5.5"])(
|
||||
"prefers a native deployment over the cloud group named %s",
|
||||
(cloudGroup) => {
|
||||
const availability = buildModelAvailability(
|
||||
[cloudGroup, "z-native"],
|
||||
[
|
||||
{ modelGroup: cloudGroup, underlyingModels: ["bedrock/us.anthropic.claude-opus-5-5-v1:0"] },
|
||||
{ modelGroup: "z-native", underlyingModels: ["anthropic/claude-opus-5-5"] },
|
||||
],
|
||||
);
|
||||
|
||||
expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("z-native");
|
||||
expect(resolveAvailableModels("claude-opus-5-5", availability)).toEqual([cloudGroup]);
|
||||
},
|
||||
);
|
||||
|
||||
it("breaks ties between native groups alphabetically regardless of deployment order", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["z-group", "a-group"],
|
||||
["z-native", "a-native"],
|
||||
[
|
||||
{ modelGroup: "z-group", underlyingModels: ["anthropic/claude-opus-5"] },
|
||||
{ modelGroup: "a-group", underlyingModels: ["bedrock/us.anthropic.claude-opus-5-v1:0"] },
|
||||
{ modelGroup: "z-native", underlyingModels: ["anthropic/claude-opus-5-5"] },
|
||||
{ modelGroup: "a-native", underlyingModels: ["anthropic/claude-opus-5-5"] },
|
||||
],
|
||||
);
|
||||
const config = {
|
||||
tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
classifier_type: "heuristic" as const,
|
||||
classification_mode: "every_request" as const,
|
||||
session_affinity: false,
|
||||
deployment_affinity: true,
|
||||
};
|
||||
expect(buildPresetPrefill(config, availability).complexityRouterConfig.tiers.SIMPLE).toEqual(["a-group"]);
|
||||
|
||||
expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("a-native");
|
||||
});
|
||||
|
||||
it("prefers an exact group-name match over the deployment index", () => {
|
||||
it.each(["gpt-6-sol", "claude-opus-5-5"])("recognizes the native default of bare %s", (model) => {
|
||||
const availability = buildModelAvailability(
|
||||
["a-cloud", "z-native"],
|
||||
[
|
||||
{ modelGroup: "a-cloud", underlyingModels: [`openrouter/${model}`] },
|
||||
{ modelGroup: "z-native", underlyingModels: [model] },
|
||||
],
|
||||
);
|
||||
|
||||
expect(resolveAvailableModel(model, availability)).toBe("z-native");
|
||||
});
|
||||
|
||||
it.each(["bedrock/claude-opus-5-5", "unknown-model"])(
|
||||
"prefers an exclusively native group over one that also routes to %s",
|
||||
(otherModel) => {
|
||||
const deployments = [
|
||||
{ modelGroup: "a-cloud", underlyingModels: ["bedrock/claude-opus-5-5"] },
|
||||
{ modelGroup: "b-mixed", underlyingModels: ["anthropic/claude-opus-5-5"] },
|
||||
{ modelGroup: "b-mixed", underlyingModels: [otherModel] },
|
||||
{ modelGroup: "z-native", underlyingModels: ["anthropic/claude-opus-5-5"] },
|
||||
];
|
||||
const availability = buildModelAvailability(["a-cloud", "b-mixed", "z-native"], deployments);
|
||||
|
||||
expect(resolveAvailableModel("claude-opus-5-5", availability)).toBe("z-native");
|
||||
const noNativeGroup = buildModelAvailability(["a-cloud", "b-mixed"], deployments);
|
||||
expect(resolveAvailableModel("claude-opus-5-5", noNativeGroup)).toBe("a-cloud");
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ model: "azure/opaque-deployment", base_model: "openai/gpt-6-sol" },
|
||||
{ model: "openai/gpt-6-sol", custom_llm_provider: "openrouter" },
|
||||
])("keeps cloud routing authoritative over native-looking model metadata: %j", (litellmParams) => {
|
||||
const availability = buildModelAvailability(
|
||||
["a-cloud", "z-native"],
|
||||
deploymentRefsFromModelInfo([
|
||||
{ model_name: "a-cloud", litellm_params: litellmParams, model_info: { base_model: "openai/gpt-6-sol" } },
|
||||
{ model_name: "z-native", litellm_params: { model: "openai/gpt-6-sol" } },
|
||||
]),
|
||||
);
|
||||
|
||||
expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("z-native");
|
||||
});
|
||||
|
||||
it("recognizes an explicit native provider on an otherwise unqualified model", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["a-cloud", "z-native"],
|
||||
deploymentRefsFromModelInfo([
|
||||
{ model_name: "a-cloud", litellm_params: { model: "openrouter/meta/muse-spark-1.3" } },
|
||||
{ model_name: "z-native", litellm_params: { model: "muse-spark-1.3", custom_llm_provider: "meta" } },
|
||||
]),
|
||||
);
|
||||
|
||||
expect(resolveAvailableModel("muse-spark-1.3", availability)).toBe("z-native");
|
||||
});
|
||||
|
||||
it("preserves exact group-name precedence when no known native deployment is available", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["claude-opus-5", "renamed-opus"],
|
||||
[{ modelGroup: "renamed-opus", underlyingModels: ["anthropic/claude-opus-5"] }],
|
||||
[{ modelGroup: "renamed-opus", underlyingModels: ["bedrock/us.anthropic.claude-opus-5-v1:0"] }],
|
||||
);
|
||||
const config = {
|
||||
tiers: { SIMPLE: ["claude-opus-5"], MEDIUM: [], COMPLEX: [], REASONING: [] },
|
||||
|
|
@ -573,6 +686,70 @@ describe("autorouter_presets", () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it.each(["native/*", "*"])("ranks wildcard groups using their routing deployment: %s", (nativePattern) => {
|
||||
const nativeGroup = nativePattern === "*" ? "openai/gpt-6-sol" : "native/gpt-6-sol";
|
||||
const availability = buildModelAvailability(
|
||||
["azure/gpt-6-sol", nativeGroup],
|
||||
[
|
||||
{ modelGroup: "azure/*", underlyingModels: ["openrouter/*"] },
|
||||
{ modelGroup: nativePattern, underlyingModels: ["openai/*"] },
|
||||
],
|
||||
);
|
||||
|
||||
expect(resolveAvailableModel("gpt-6-sol", availability)).toBe(nativeGroup);
|
||||
});
|
||||
|
||||
it("does not treat a native-looking wildcard group as native when its deployment uses the cloud", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["openai/gpt-6-sol", "z-native/gpt-6-sol"],
|
||||
[
|
||||
{ modelGroup: "openai/*", underlyingModels: ["azure/*"] },
|
||||
{ modelGroup: "z-native/*", underlyingModels: ["openai/*"] },
|
||||
],
|
||||
);
|
||||
|
||||
expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("z-native/gpt-6-sol");
|
||||
});
|
||||
|
||||
it("keeps literal native deployments ahead of a matching cloud wildcard", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["a-cloud", "team/gpt-6-sol"],
|
||||
[
|
||||
{ modelGroup: "a-cloud", underlyingModels: ["azure/gpt-6-sol"] },
|
||||
{ modelGroup: "team/gpt-6-sol", underlyingModels: ["openai/gpt-6-sol"] },
|
||||
{ modelGroup: "team/*", underlyingModels: ["azure/*"] },
|
||||
],
|
||||
);
|
||||
|
||||
expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("team/gpt-6-sol");
|
||||
});
|
||||
|
||||
it("does not promote a bare-star expansion when its routing group also contains a cloud deployment", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["openai/gpt-6-sol", "z-native"],
|
||||
[
|
||||
{ modelGroup: "*", underlyingModels: ["openai/*"] },
|
||||
{ modelGroup: "*", underlyingModels: ["azure/*"] },
|
||||
{ modelGroup: "z-native", underlyingModels: ["openai/gpt-6-sol"] },
|
||||
],
|
||||
);
|
||||
|
||||
expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("z-native");
|
||||
});
|
||||
|
||||
it("retains fallback ordering when overlapping wildcard routes have different providers", () => {
|
||||
const availability = buildModelAvailability(
|
||||
["a-cloud", "team/gpt-6-sol"],
|
||||
[
|
||||
{ modelGroup: "a-cloud", underlyingModels: ["azure/gpt-6-sol"] },
|
||||
{ modelGroup: "team/*", underlyingModels: ["azure/*"] },
|
||||
{ modelGroup: "team/gpt-*", underlyingModels: ["openai/gpt-*"] },
|
||||
],
|
||||
);
|
||||
|
||||
expect(resolveAvailableModel("gpt-6-sol", availability)).toBe("a-cloud");
|
||||
});
|
||||
|
||||
it.each(getAllPresets().map((preset) => [preset.key, preset] as const))(
|
||||
"fully resolves the %s preset through wildcard-expanded groups only",
|
||||
(_key, preset) => {
|
||||
|
|
@ -602,7 +779,9 @@ describe("autorouter_presets", () => {
|
|||
{ model_name: "no-underlying", litellm_params: {}, model_info: {} },
|
||||
{ litellm_params: { model: "openai/gpt-5.4" } },
|
||||
]);
|
||||
expect(refs).toEqual([{ modelGroup: "azure-prod", underlyingModels: ["azure/my-deployment", "azure/gpt-5.4"] }]);
|
||||
expect(refs).toEqual([
|
||||
{ modelGroup: "azure-prod", underlyingModels: ["azure/my-deployment", "azure/gpt-5.4"], provider: "azure" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("lets an azure deployment resolve through base_model declared under litellm_params", () => {
|
||||
|
|
|
|||
|
|
@ -65,13 +65,37 @@ export const normalizeModelName = (model: string): string => model.replace(/(\d)
|
|||
export interface DeploymentModelRef {
|
||||
modelGroup: string;
|
||||
underlyingModels: readonly string[];
|
||||
provider?: string;
|
||||
}
|
||||
|
||||
export interface ModelAvailability {
|
||||
modelGroups: Set<string>;
|
||||
underlyingIndex: Map<string, readonly string[]>;
|
||||
nativeUnderlyingIndex: Map<string, readonly string[]>;
|
||||
}
|
||||
|
||||
const NATIVE_MODEL_PROVIDERS: readonly (readonly [RegExp, string])[] = [
|
||||
[/^(gpt-|o\d|text-embedding-)/, "openai"],
|
||||
[/^claude-/, "anthropic"],
|
||||
[/^gemini-/, "gemini"],
|
||||
[/^deepseek-/, "deepseek"],
|
||||
[/^muse-/, "meta"],
|
||||
[/^kimi-/, "moonshot"],
|
||||
[/^grok-/, "xai"],
|
||||
];
|
||||
|
||||
const nativeModelProvider = (model: string): string | undefined =>
|
||||
NATIVE_MODEL_PROVIDERS.find(([pattern]) => pattern.test(model))?.[1];
|
||||
|
||||
const routingProvider = (model: string): string => {
|
||||
if (model.includes("/")) return model.split("/")[0];
|
||||
const native = nativeModelProvider(model);
|
||||
return native === "openai" || native === "anthropic" ? native : "";
|
||||
};
|
||||
|
||||
const deploymentProvider = (deployment: DeploymentModelRef): string =>
|
||||
deployment.provider ?? routingProvider(deployment.underlyingModels[0] ?? "");
|
||||
|
||||
const normalizeUnderlyingModel = (model: string): string | null => {
|
||||
if (model.includes("*")) return null;
|
||||
const ownName = model.slice(model.lastIndexOf("/") + 1).split("@")[0];
|
||||
|
|
@ -107,32 +131,42 @@ export const buildModelAvailability = (
|
|||
deployments: readonly DeploymentModelRef[],
|
||||
): ModelAvailability => {
|
||||
const groups = new Set(modelGroups);
|
||||
const deploymentGroups = new Set(deployments.map((deployment) => deployment.modelGroup));
|
||||
const deploymentProviders = new Map<string, Set<string>>();
|
||||
for (const deployment of deployments) {
|
||||
const providers = deploymentProviders.get(deployment.modelGroup) ?? new Set<string>();
|
||||
providers.add(deploymentProvider(deployment));
|
||||
deploymentProviders.set(deployment.modelGroup, providers);
|
||||
}
|
||||
const literalEntries = deployments
|
||||
.filter((deployment) => groups.has(deployment.modelGroup))
|
||||
.flatMap((deployment) =>
|
||||
deployment.underlyingModels
|
||||
.map(normalizeUnderlyingModel)
|
||||
.filter((key): key is string => key !== null)
|
||||
.map((key) => ({ key, modelGroup: deployment.modelGroup })),
|
||||
.map((key) => ({ key, modelGroup: deployment.modelGroup, sourceGroup: deployment.modelGroup })),
|
||||
);
|
||||
// Mirrors get_known_models_from_wildcard: a bare "*" model_name expands via its underlying
|
||||
// wildcard (or not at all), and a wildcard without a "/" expands to nothing.
|
||||
const wildcardPatterns = Array.from(
|
||||
new Set(
|
||||
deployments
|
||||
.flatMap((deployment) =>
|
||||
deployment.modelGroup === "*" ? deployment.underlyingModels : [deployment.modelGroup],
|
||||
)
|
||||
.filter((pattern) => pattern !== "*" && pattern.includes("*") && pattern.includes("/")),
|
||||
),
|
||||
const wildcardPatterns = deployments.flatMap((deployment) =>
|
||||
(deployment.modelGroup === "*" ? deployment.underlyingModels : [deployment.modelGroup])
|
||||
.filter((pattern) => pattern !== "*" && pattern.includes("*") && pattern.includes("/"))
|
||||
.map((pattern) => ({ pattern, sourceGroup: deployment.modelGroup })),
|
||||
);
|
||||
const wildcardEntries = Array.from(groups)
|
||||
.filter((group) => !group.includes("*") && wildcardPatterns.some((pattern) => matchesWildcard(pattern, group)))
|
||||
.map((group) => ({ key: normalizeUnderlyingModel(group), modelGroup: group }))
|
||||
.filter((entry): entry is { key: string; modelGroup: string } => entry.key !== null);
|
||||
.filter((group) => !group.includes("*") && !deploymentGroups.has(group))
|
||||
.flatMap((group) =>
|
||||
wildcardPatterns
|
||||
.filter(({ pattern }) => matchesWildcard(pattern, group))
|
||||
.map(({ sourceGroup }) => ({ key: normalizeUnderlyingModel(group), modelGroup: group, sourceGroup })),
|
||||
);
|
||||
const entries = [...literalEntries, ...wildcardEntries];
|
||||
const grouped = new Map<string, Set<string>>();
|
||||
const providersByGroup = new Map<string, Set<string>>();
|
||||
for (const entry of entries) {
|
||||
const providers = providersByGroup.get(entry.modelGroup) ?? new Set<string>();
|
||||
for (const provider of deploymentProviders.get(entry.sourceGroup) ?? []) providers.add(provider);
|
||||
providersByGroup.set(entry.modelGroup, providers);
|
||||
if (entry.key === null) continue;
|
||||
const groupsForKey = grouped.get(entry.key) ?? new Set<string>();
|
||||
groupsForKey.add(entry.modelGroup);
|
||||
grouped.set(entry.key, groupsForKey);
|
||||
|
|
@ -140,13 +174,23 @@ export const buildModelAvailability = (
|
|||
const underlyingIndex = new Map(
|
||||
Array.from(grouped, ([key, groupsForKey]) => [key, Array.from(groupsForKey).sort()] as const),
|
||||
);
|
||||
return { modelGroups: groups, underlyingIndex };
|
||||
const nativeUnderlyingIndex = new Map(
|
||||
Array.from(underlyingIndex, ([key, matches]) => [
|
||||
key,
|
||||
matches.filter((group) => {
|
||||
const native = nativeModelProvider(key);
|
||||
const providers = providersByGroup.get(group);
|
||||
return native !== undefined && providers?.size === 1 && providers.has(native);
|
||||
}),
|
||||
]),
|
||||
);
|
||||
return { modelGroups: groups, underlyingIndex, nativeUnderlyingIndex };
|
||||
};
|
||||
|
||||
export const deploymentRefsFromModelInfo = (
|
||||
rows: readonly {
|
||||
model_name?: string | null;
|
||||
litellm_params?: { model?: string | null; base_model?: string | null } | null;
|
||||
litellm_params?: { model?: string | null; base_model?: string | null; custom_llm_provider?: string | null } | null;
|
||||
model_info?: { base_model?: string | null } | null;
|
||||
}[],
|
||||
): DeploymentModelRef[] =>
|
||||
|
|
@ -156,7 +200,10 @@ export const deploymentRefsFromModelInfo = (
|
|||
row.litellm_params?.base_model,
|
||||
row.model_info?.base_model,
|
||||
].filter((model): model is string => Boolean(model));
|
||||
return row.model_name && underlyingModels.length > 0 ? [{ modelGroup: row.model_name, underlyingModels }] : [];
|
||||
const provider = row.litellm_params?.custom_llm_provider || routingProvider(row.litellm_params?.model ?? "");
|
||||
return row.model_name && underlyingModels.length > 0
|
||||
? [{ modelGroup: row.model_name, underlyingModels, provider }]
|
||||
: [];
|
||||
});
|
||||
|
||||
export const resolveAvailableModels = (requiredModel: string, availability: ModelAvailability): readonly string[] => {
|
||||
|
|
@ -169,8 +216,12 @@ export const resolveAvailableModels = (requiredModel: string, availability: Mode
|
|||
return key === null ? [] : underlyingIndex.get(key) ?? [];
|
||||
};
|
||||
|
||||
export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined =>
|
||||
resolveAvailableModels(requiredModel, availability)[0];
|
||||
export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => {
|
||||
const key = normalizeUnderlyingModel(requiredModel);
|
||||
const nativeMatches = key === null ? [] : availability.nativeUnderlyingIndex.get(key) ?? [];
|
||||
const matches = resolveAvailableModels(requiredModel, availability);
|
||||
return matches.find((model) => nativeMatches.includes(model)) ?? nativeMatches[0] ?? matches[0];
|
||||
};
|
||||
|
||||
export const getMissingModels = (
|
||||
config: Parameters<typeof getRequiredModels>[0],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue