mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
feat(ui): prefer proven models in Auto Setup fallback
This commit is contained in:
parent
a48afd2242
commit
72ddd699dd
5 changed files with 136 additions and 8 deletions
|
|
@ -210,6 +210,25 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(toast.success).toHaveBeenCalledWith(`Configured with ${openAiPreset.label}`);
|
||||
});
|
||||
|
||||
it("mixes available models from the preferred tier catalog when no complete template fits", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue(
|
||||
["gpt-5.6-luna", "claude-sonnet-5", "gpt-5.6-sol"].map((model_group) => ({
|
||||
model_group,
|
||||
mode: "chat",
|
||||
})),
|
||||
);
|
||||
mockFetchAllModelDeployments.mockResolvedValue([]);
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
const button = screen.getByTestId("configure-automatically-button");
|
||||
await waitFor(() => expect(button).toBeEnabled());
|
||||
await userEvent.click(button);
|
||||
|
||||
expect(
|
||||
screen.getByText(/Simple: gpt-5.6-luna.*Medium: claude-sonnet-5.*Complex: gpt-5.6-sol.*Reasoning: gpt-5.6-sol/),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// Nothing is filled in, so there is nothing to submit. The button reports that itself instead of
|
||||
// accepting a click and answering with a toast.
|
||||
it("offers no submit at all until every tier has a model", async () => {
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ import {
|
|||
} from "@/lib/autorouter_presets";
|
||||
import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets";
|
||||
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
|
||||
import { buildAutomaticRouterConfig } from "./auto_setup";
|
||||
import { buildAutomaticRouterConfig, buildPreferredTierModels } from "./auto_setup";
|
||||
|
||||
interface AddAutoRouterTabProps {
|
||||
handleOk: () => void;
|
||||
|
|
@ -319,9 +319,11 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
};
|
||||
|
||||
const handleAutomaticSetup = () => {
|
||||
const matchingPreset = AUTO_SETUP_PRESET_PRIORITY.map((key) => presets.find((preset) => preset.key === key)).find(
|
||||
(preset) => preset && presetAvailability(preset).kind === "available",
|
||||
);
|
||||
const prioritizedPresets = AUTO_SETUP_PRESET_PRIORITY.flatMap((key) => {
|
||||
const preset = presets.find((candidate) => candidate.key === key);
|
||||
return preset ? [preset] : [];
|
||||
});
|
||||
const matchingPreset = prioritizedPresets.find((preset) => presetAvailability(preset).kind === "available");
|
||||
if (matchingPreset) {
|
||||
const presetState = presetAvailability(matchingPreset);
|
||||
setSelectedPreset(matchingPreset.key);
|
||||
|
|
@ -331,7 +333,8 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
return;
|
||||
}
|
||||
|
||||
const generatedConfig = buildAutomaticRouterConfig(modelInfo, deployments ?? [], modelCostMap);
|
||||
const preferredTierModels = buildPreferredTierModels(prioritizedPresets, availability);
|
||||
const generatedConfig = buildAutomaticRouterConfig(modelInfo, deployments ?? [], modelCostMap, preferredTierModels);
|
||||
if (generatedConfig === null) {
|
||||
toast.fromError("Add at least one chat model before configuring an Auto Router");
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import { buildAutomaticRouterConfig } from "./auto_setup";
|
||||
import { buildAutomaticRouterConfig, type PreferredTierModels } from "./auto_setup";
|
||||
|
||||
const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" }));
|
||||
|
||||
|
|
@ -24,6 +24,68 @@ const tierModels = (config: ReturnType<typeof buildAutomaticRouterConfig>) =>
|
|||
];
|
||||
|
||||
describe("buildAutomaticRouterConfig", () => {
|
||||
it("uses available preferred models before price ranking", () => {
|
||||
const preferred: PreferredTierModels = {
|
||||
SIMPLE: ["preferred-simple"],
|
||||
MEDIUM: ["preferred-medium"],
|
||||
COMPLEX: ["preferred-complex"],
|
||||
REASONING: ["preferred-reasoning"],
|
||||
};
|
||||
const available = [...Object.values(preferred).flat(), "cheap-decoy", "expensive-decoy"];
|
||||
const config = buildAutomaticRouterConfig(
|
||||
models(...available),
|
||||
available.map((name, index) => deployment(name, index + 1)),
|
||||
{},
|
||||
preferred,
|
||||
);
|
||||
|
||||
expect(tierModels(config)).toEqual([
|
||||
"preferred-simple",
|
||||
"preferred-medium",
|
||||
"preferred-complex",
|
||||
"preferred-reasoning",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reuses the nearest preferred model for tiers with no preferred match", () => {
|
||||
const preferred: PreferredTierModels = {
|
||||
SIMPLE: ["preferred-simple"],
|
||||
MEDIUM: [],
|
||||
COMPLEX: ["preferred-complex"],
|
||||
REASONING: [],
|
||||
};
|
||||
const config = buildAutomaticRouterConfig(
|
||||
models("preferred-simple", "preferred-complex", "cheap-decoy"),
|
||||
[deployment("preferred-simple", 4), deployment("preferred-complex", 5), deployment("cheap-decoy", 1)],
|
||||
{},
|
||||
preferred,
|
||||
);
|
||||
|
||||
expect(tierModels(config)).toEqual([
|
||||
"preferred-simple",
|
||||
"preferred-simple",
|
||||
"preferred-complex",
|
||||
"preferred-complex",
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses price ranking when none of the preferred models are available", () => {
|
||||
const unavailablePreferred: PreferredTierModels = {
|
||||
SIMPLE: ["missing-simple"],
|
||||
MEDIUM: ["missing-medium"],
|
||||
COMPLEX: ["missing-complex"],
|
||||
REASONING: ["missing-reasoning"],
|
||||
};
|
||||
const config = buildAutomaticRouterConfig(
|
||||
models("expensive", "cheap", "premium", "middle"),
|
||||
[deployment("cheap", 1), deployment("middle", 2), deployment("premium", 3), deployment("expensive", 4)],
|
||||
{},
|
||||
unavailablePreferred,
|
||||
);
|
||||
|
||||
expect(tierModels(config)).toEqual(["cheap", "middle", "premium", "expensive"]);
|
||||
});
|
||||
|
||||
it("uses four different models when four are available", () => {
|
||||
const config = buildAutomaticRouterConfig(
|
||||
models("expensive", "cheap", "premium", "middle"),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { isAutoRouterDeployment, type AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import type { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { resolveAvailableModel, type AutoRouterPreset, type ModelAvailability } from "@/lib/autorouter_presets";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
|
||||
type ModelCost = {
|
||||
|
|
@ -9,6 +10,10 @@ type ModelCost = {
|
|||
|
||||
export type ModelCostMap = Record<string, ModelCost>;
|
||||
|
||||
const TIER_NAMES = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const;
|
||||
type TierName = (typeof TIER_NAMES)[number];
|
||||
export type PreferredTierModels = Record<TierName, string[]>;
|
||||
|
||||
const price = (cost: ModelCost | null | undefined): number | undefined => {
|
||||
const input = cost?.input_cost_per_token;
|
||||
const output = cost?.output_cost_per_token;
|
||||
|
|
@ -56,10 +61,46 @@ const selectTierModels = (ranked: string[]): [string, string, string, string] =>
|
|||
return [ranked[0], ranked[Math.floor(last / 3)], ranked[Math.floor((2 * last) / 3)], ranked[last]];
|
||||
};
|
||||
|
||||
export const buildPreferredTierModels = (
|
||||
presets: AutoRouterPreset[],
|
||||
availability: ModelAvailability,
|
||||
): PreferredTierModels =>
|
||||
Object.fromEntries(
|
||||
TIER_NAMES.map((tier) => [
|
||||
tier,
|
||||
Array.from(
|
||||
new Set(
|
||||
presets.flatMap((preset) =>
|
||||
preset.complexity_router_config.tiers[tier].flatMap((model) => {
|
||||
const resolved = resolveAvailableModel(model, availability);
|
||||
return resolved ? [resolved] : [];
|
||||
}),
|
||||
),
|
||||
),
|
||||
),
|
||||
]),
|
||||
) as PreferredTierModels;
|
||||
|
||||
const selectPreferredTierModels = (
|
||||
preferredByTier: PreferredTierModels,
|
||||
usableNames: ReadonlySet<string>,
|
||||
): [string, string, string, string] | null => {
|
||||
const preferred = TIER_NAMES.map((tier) => preferredByTier[tier].find((name) => usableNames.has(name)));
|
||||
const candidates = preferred.flatMap((model, tier) => (model ? [{ model, tier }] : []));
|
||||
if (candidates.length === 0) return null;
|
||||
|
||||
const nearest = (tier: number): string =>
|
||||
[...candidates].sort(
|
||||
(left, right) => Math.abs(left.tier - tier) - Math.abs(right.tier - tier) || left.tier - right.tier,
|
||||
)[0].model;
|
||||
return preferred.map((model, tier) => model ?? nearest(tier)) as [string, string, string, string];
|
||||
};
|
||||
|
||||
export const buildAutomaticRouterConfig = (
|
||||
models: ModelGroup[],
|
||||
deployments: AutoRouterDeployment[],
|
||||
costMap: ModelCostMap,
|
||||
preferredByTier?: PreferredTierModels,
|
||||
): ComplexityRouterConfigValue | null => {
|
||||
const autoRouterNames: ReadonlySet<string> = new Set(
|
||||
deployments
|
||||
|
|
@ -75,6 +116,7 @@ export const buildAutomaticRouterConfig = (
|
|||
),
|
||||
);
|
||||
if (names.length === 0) return null;
|
||||
const usableNames: ReadonlySet<string> = new Set(names);
|
||||
|
||||
const ranked = names
|
||||
.map((name) => ({ name, price: groupPrice(name, deployments, costMap) }))
|
||||
|
|
@ -88,7 +130,9 @@ export const buildAutomaticRouterConfig = (
|
|||
})
|
||||
.map(({ name }) => name);
|
||||
|
||||
const selected = selectTierModels(ranked);
|
||||
const selected = preferredByTier
|
||||
? selectPreferredTierModels(preferredByTier, usableNames) ?? selectTierModels(ranked)
|
||||
: selectTierModels(ranked);
|
||||
|
||||
return {
|
||||
tiers: {
|
||||
|
|
|
|||
|
|
@ -152,7 +152,7 @@ export const deploymentRefsFromModelInfo = (
|
|||
return row.model_name && underlyingModels.length > 0 ? [{ modelGroup: row.model_name, underlyingModels }] : [];
|
||||
});
|
||||
|
||||
const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => {
|
||||
export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => {
|
||||
const { modelGroups, underlyingIndex } = availability;
|
||||
if (modelGroups.has(requiredModel)) return requiredModel;
|
||||
const normalized = normalizeModelName(requiredModel);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue