mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-06 08:16:43 +00:00
feat(ui): add one-click Auto Router setup
This commit is contained in:
parent
4e18c0f63a
commit
940fdfb26b
5 changed files with 304 additions and 0 deletions
|
|
@ -90,6 +90,9 @@ export interface AutoRouterCandidateDeployment {
|
|||
export interface AutoRouterDeployment extends AutoRouterCandidateDeployment {
|
||||
litellm_params?: {
|
||||
model?: string | null;
|
||||
base_model?: string | null;
|
||||
input_cost_per_token?: number | null;
|
||||
output_cost_per_token?: number | null;
|
||||
complexity_router_config?: unknown;
|
||||
complexity_router_default_model?: string | null;
|
||||
auto_router_config?: unknown;
|
||||
|
|
@ -105,6 +108,8 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment {
|
|||
/** False for config.yaml-defined deployments, which the update and delete routes refuse. */
|
||||
db_model?: boolean | null;
|
||||
base_model?: string | null;
|
||||
input_cost_per_token?: number | null;
|
||||
output_cost_per_token?: number | null;
|
||||
created_at?: string | null;
|
||||
updated_at?: string | null;
|
||||
team_id?: string | null;
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@ vi.mock(
|
|||
"@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets",
|
||||
async () => await import("../../../tests/mocks/autoRouterPresets"),
|
||||
);
|
||||
vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
|
||||
useModelCostMap: () => ({ data: {}, isLoading: false }),
|
||||
}));
|
||||
|
||||
const getAllPresets = (): AutoRouterPreset[] => BUNDLED_PRESETS;
|
||||
const getPresetByKey = (key: string): AutoRouterPreset | undefined => BUNDLED_PRESETS.find((p) => p.key === key);
|
||||
|
|
@ -155,6 +158,58 @@ describe("AddAutoRouterTab", () => {
|
|||
expect(screen.getByText("Complexity Tier Configuration")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("configures four unique tiers from the available models with one click", async () => {
|
||||
mockFetchAvailableModels.mockResolvedValue([
|
||||
{ model_group: "premium", mode: "chat" },
|
||||
{ model_group: "cheap", mode: "chat" },
|
||||
{ model_group: "best", mode: "chat" },
|
||||
{ model_group: "middle", mode: "chat" },
|
||||
]);
|
||||
mockFetchAllModelDeployments.mockResolvedValue([
|
||||
{ model_name: "cheap", litellm_params: { model: "cheap", input_cost_per_token: 1 } },
|
||||
{ model_name: "middle", litellm_params: { model: "middle", input_cost_per_token: 2 } },
|
||||
{ model_name: "premium", litellm_params: { model: "premium", input_cost_per_token: 3 } },
|
||||
{ model_name: "best", litellm_params: { model: "best", input_cost_per_token: 4 } },
|
||||
]);
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
const button = screen.getByTestId("configure-automatically-button");
|
||||
await waitFor(() => expect(button).toBeEnabled());
|
||||
await userEvent.click(button);
|
||||
|
||||
expect(screen.getByText(/Simple: cheap.*Medium: middle.*Complex: premium.*Reasoning: best/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("prefers the first compatible bundled template over the price fallback", async () => {
|
||||
const firstPreset = getAllPresets()[0];
|
||||
mockFetchAvailableModels.mockResolvedValue(
|
||||
[...getRequiredModelsInPreset(firstPreset)].map((model_group) => ({ model_group, mode: "chat" })),
|
||||
);
|
||||
mockFetchAllModelDeployments.mockResolvedValue([]);
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
const button = screen.getByTestId("configure-automatically-button");
|
||||
await waitFor(() => expect(button).toBeEnabled());
|
||||
await userEvent.click(button);
|
||||
|
||||
expect(toast.success).toHaveBeenCalledWith(`Configured with ${firstPreset.label}`);
|
||||
});
|
||||
|
||||
it("prefers the OpenAI template over Gemini", async () => {
|
||||
const openAiPreset = getPresetByKey("openai_family")!;
|
||||
const geminiPreset = getPresetByKey("gemini_family")!;
|
||||
const available = new Set([...getRequiredModelsInPreset(openAiPreset), ...getRequiredModelsInPreset(geminiPreset)]);
|
||||
mockFetchAvailableModels.mockResolvedValue([...available].map((model_group) => ({ model_group, mode: "chat" })));
|
||||
mockFetchAllModelDeployments.mockResolvedValue([]);
|
||||
renderWithProviders(<Harness />);
|
||||
|
||||
const button = screen.getByTestId("configure-automatically-button");
|
||||
await waitFor(() => expect(button).toBeEnabled());
|
||||
await userEvent.click(button);
|
||||
|
||||
expect(toast.success).toHaveBeenCalledWith(`Configured with ${openAiPreset.label}`);
|
||||
});
|
||||
|
||||
// Nothing is filled in, so there is nothing to submit. The button reports that itself instead of
|
||||
// accepting a click and answering with a toast.
|
||||
it("offers no submit at all until every tier has a model", async () => {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import TeamDropdown from "../common_components/team_dropdown";
|
|||
import { type AddAutoRouterValues, handleAddAutoRouterSubmit } from "./handle_add_auto_router_submit";
|
||||
import { fetchAvailableModels, type ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import { autoRouterListKey, fetchAllModelDeployments } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
|
||||
import ComplexityRouterConfig, {
|
||||
ComplexityRouterConfigValue,
|
||||
effectiveClassifierType,
|
||||
|
|
@ -64,6 +65,7 @@ import {
|
|||
} from "@/lib/autorouter_presets";
|
||||
import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets";
|
||||
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
|
||||
import { buildAutomaticRouterConfig } from "./auto_setup";
|
||||
|
||||
interface AddAutoRouterTabProps {
|
||||
handleOk: () => void;
|
||||
|
|
@ -104,6 +106,7 @@ const presetDisabledHint = (availability: PresetAvailability): string | null =>
|
|||
const isPresetHintAlarming = (availability: PresetAvailability): boolean => availability.kind === "missing_models";
|
||||
|
||||
const NO_PRESETS: AutoRouterPreset[] = [];
|
||||
const AUTO_SETUP_PRESET_PRIORITY = ["1m_context", "anthropic_family", "openai_family", "gemini_family", "lite"];
|
||||
|
||||
// A one-line summary of what's configured, shown when the detailed section is collapsed so a
|
||||
// caller can see the shape of the config without opening it.
|
||||
|
|
@ -233,6 +236,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
queryFn: () => fetchAllModelDeployments(accessToken, userId ?? "", userRole),
|
||||
enabled: Boolean(accessToken),
|
||||
});
|
||||
const { data: modelCostMap = {}, isLoading: costsLoading } = useModelCostMap();
|
||||
const modelsLoading = groupsLoading || deploymentsLoading;
|
||||
const modelInfo = React.useMemo(() => data ?? [], [data]);
|
||||
const {
|
||||
|
|
@ -242,6 +246,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
refetch: refetchPresets,
|
||||
} = useAutoRouterPresets();
|
||||
const presets = presetsData ?? NO_PRESETS;
|
||||
const automaticSetupLoading = modelsLoading || costsLoading || presetsPending;
|
||||
const presetsUnavailable = presetsError && presetsData === undefined;
|
||||
// react-query keeps the last successful list around when a later refetch fails, so isError alone
|
||||
// can't tell "never loaded" apart from "loaded, then a background refetch errored" - only the
|
||||
|
|
@ -313,6 +318,30 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
setEscalationKeywords(prefill.escalationKeywords);
|
||||
};
|
||||
|
||||
const handleAutomaticSetup = () => {
|
||||
const matchingPreset = AUTO_SETUP_PRESET_PRIORITY.map((key) => presets.find((preset) => preset.key === key)).find(
|
||||
(preset) => preset && presetAvailability(preset).kind === "available",
|
||||
);
|
||||
if (matchingPreset) {
|
||||
const presetState = presetAvailability(matchingPreset);
|
||||
setSelectedPreset(matchingPreset.key);
|
||||
applyPrefill(buildPresetPrefill(matchingPreset.complexity_router_config, availability));
|
||||
setDetailsExpanded(presetState.kind === "available" && presetState.viaDeployments);
|
||||
toast.success(`Configured with ${matchingPreset.label}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const generatedConfig = buildAutomaticRouterConfig(modelInfo, deployments ?? [], modelCostMap);
|
||||
if (generatedConfig === null) {
|
||||
toast.fromError("Add at least one chat model before configuring an Auto Router");
|
||||
return;
|
||||
}
|
||||
setSelectedPreset(undefined);
|
||||
applyPrefill({ ...buildEmptyPrefill(), complexityRouterConfig: generatedConfig });
|
||||
setDetailsExpanded(false);
|
||||
toast.success("Automatic setup created", { description: tierConfigSummary(generatedConfig) });
|
||||
};
|
||||
|
||||
const handlePresetChange = (presetKey: string | undefined) => {
|
||||
if (!presetKey || presetKey === "custom") {
|
||||
setSelectedPreset(presetKey);
|
||||
|
|
@ -508,6 +537,25 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
|
|||
{({ ref, ...field }) => <Input {...field} ref={ref} placeholder="e.g., smart_router, auto_router_1" />}
|
||||
</FormField>
|
||||
|
||||
<div className="flex items-center justify-between gap-4 rounded-lg border border-border p-4">
|
||||
<div>
|
||||
<div className="text-sm font-medium text-foreground">Start with a recommended setup</div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
Uses your available models and their listed prices. You can review and edit everything before
|
||||
saving.
|
||||
</div>
|
||||
</div>
|
||||
<Button
|
||||
type="button"
|
||||
data-testid="configure-automatically-button"
|
||||
disabled={automaticSetupLoading}
|
||||
onClick={handleAutomaticSetup}
|
||||
>
|
||||
{automaticSetupLoading && <UiLoadingSpinner className="size-4" />}
|
||||
Configure automatically
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block text-sm font-medium text-foreground mb-2">Template</label>
|
||||
<Select
|
||||
|
|
|
|||
101
ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts
Normal file
101
ui/litellm-dashboard/src/components/add_model/auto_setup.test.ts
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import { buildAutomaticRouterConfig } from "./auto_setup";
|
||||
|
||||
const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" }));
|
||||
|
||||
const deployment = (name: string, cost: number): AutoRouterDeployment => ({
|
||||
model_name: name,
|
||||
litellm_params: {
|
||||
model: name,
|
||||
input_cost_per_token: cost / 2,
|
||||
output_cost_per_token: cost / 2,
|
||||
},
|
||||
});
|
||||
|
||||
const firstModel = (value: string | string[]): string => (typeof value === "string" ? value : value[0]);
|
||||
|
||||
const tierModels = (config: ReturnType<typeof buildAutomaticRouterConfig>) =>
|
||||
config && [
|
||||
firstModel(config.tiers.SIMPLE),
|
||||
firstModel(config.tiers.MEDIUM),
|
||||
firstModel(config.tiers.COMPLEX),
|
||||
firstModel(config.tiers.REASONING),
|
||||
];
|
||||
|
||||
describe("buildAutomaticRouterConfig", () => {
|
||||
it("uses four different models when four are available", () => {
|
||||
const config = buildAutomaticRouterConfig(
|
||||
models("expensive", "cheap", "premium", "middle"),
|
||||
[deployment("cheap", 1), deployment("middle", 2), deployment("premium", 3), deployment("expensive", 4)],
|
||||
{},
|
||||
);
|
||||
|
||||
expect(tierModels(config)).toEqual(["cheap", "middle", "premium", "expensive"]);
|
||||
expect(config?.classifier_type).toBe("heuristic_v2");
|
||||
});
|
||||
|
||||
it("only repeats models when fewer than four are available", () => {
|
||||
expect(
|
||||
tierModels(
|
||||
buildAutomaticRouterConfig(
|
||||
models("cheap", "expensive"),
|
||||
[deployment("cheap", 1), deployment("expensive", 4)],
|
||||
{},
|
||||
),
|
||||
),
|
||||
).toEqual(["cheap", "cheap", "expensive", "expensive"]);
|
||||
});
|
||||
|
||||
it("uses the published cost map when deployments do not define prices", () => {
|
||||
const config = buildAutomaticRouterConfig(
|
||||
models("premium", "cheap", "middle"),
|
||||
[
|
||||
{ model_name: "premium", litellm_params: { model: "provider/premium" } },
|
||||
{ model_name: "cheap", litellm_params: { model: "provider/cheap" } },
|
||||
{ model_name: "middle", litellm_params: { model: "provider/middle" } },
|
||||
],
|
||||
{
|
||||
"provider/cheap": { input_cost_per_token: 1, output_cost_per_token: 1 },
|
||||
"provider/middle": { input_cost_per_token: 2, output_cost_per_token: 2 },
|
||||
"provider/premium": { input_cost_per_token: 3, output_cost_per_token: 3 },
|
||||
},
|
||||
);
|
||||
|
||||
expect(tierModels(config)).toEqual(["cheap", "middle", "premium", "premium"]);
|
||||
});
|
||||
|
||||
it("uses the most expensive deployment when a group has several", () => {
|
||||
const config = buildAutomaticRouterConfig(
|
||||
models("variable", "steady", "premium", "top"),
|
||||
[
|
||||
deployment("variable", 1),
|
||||
deployment("variable", 8),
|
||||
deployment("steady", 2),
|
||||
deployment("premium", 3),
|
||||
deployment("top", 4),
|
||||
],
|
||||
{},
|
||||
);
|
||||
|
||||
expect(tierModels(config)).toEqual(["steady", "premium", "top", "variable"]);
|
||||
});
|
||||
|
||||
it("ignores non-chat and existing auto-router models", () => {
|
||||
const config = buildAutomaticRouterConfig(
|
||||
[
|
||||
{ model_group: "chat-model", mode: "chat" },
|
||||
{ model_group: "image-model", mode: "image_generation" },
|
||||
{ model_group: "auto_router/existing", mode: "chat" },
|
||||
],
|
||||
[deployment("chat-model", 1)],
|
||||
{},
|
||||
);
|
||||
|
||||
expect(tierModels(config)).toEqual(["chat-model", "chat-model", "chat-model", "chat-model"]);
|
||||
});
|
||||
|
||||
it("returns null when there are no usable models", () => {
|
||||
expect(buildAutomaticRouterConfig([], [], {})).toBeNull();
|
||||
});
|
||||
});
|
||||
95
ui/litellm-dashboard/src/components/add_model/auto_setup.ts
Normal file
95
ui/litellm-dashboard/src/components/add_model/auto_setup.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import type { ModelGroup } from "@/components/llm_calls/fetch_models";
|
||||
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
|
||||
|
||||
type ModelCost = {
|
||||
input_cost_per_token?: number | null;
|
||||
output_cost_per_token?: number | null;
|
||||
};
|
||||
|
||||
export type ModelCostMap = Record<string, ModelCost>;
|
||||
|
||||
const price = (cost: ModelCost | null | undefined): number | undefined => {
|
||||
const input = cost?.input_cost_per_token;
|
||||
const output = cost?.output_cost_per_token;
|
||||
if (typeof input !== "number" && typeof output !== "number") return undefined;
|
||||
return (input ?? 0) + (output ?? 0);
|
||||
};
|
||||
|
||||
const deploymentPrice = (deployment: AutoRouterDeployment, costMap: ModelCostMap): number | undefined => {
|
||||
const configured = price(deployment.litellm_params) ?? price(deployment.model_info);
|
||||
if (configured !== undefined) return configured;
|
||||
|
||||
const references = [
|
||||
deployment.litellm_params?.model,
|
||||
deployment.litellm_params?.base_model,
|
||||
deployment.model_info?.base_model,
|
||||
];
|
||||
for (const reference of references) {
|
||||
if (reference && costMap[reference]) return price(costMap[reference]);
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const groupPrice = (
|
||||
modelGroup: string,
|
||||
deployments: AutoRouterDeployment[],
|
||||
costMap: ModelCostMap,
|
||||
): number | undefined => {
|
||||
const groupDeployments = deployments.filter(
|
||||
(deployment) =>
|
||||
deployment.model_name === modelGroup && !deployment.litellm_params?.model?.startsWith("auto_router/"),
|
||||
);
|
||||
if (groupDeployments.length === 0) return price(costMap[modelGroup]);
|
||||
const prices = groupDeployments.map((deployment) => deploymentPrice(deployment, costMap));
|
||||
if (prices.some((value) => value === undefined)) return undefined;
|
||||
const knownPrices = prices.filter((value): value is number => value !== undefined);
|
||||
return Math.max(...knownPrices);
|
||||
};
|
||||
|
||||
export const buildAutomaticRouterConfig = (
|
||||
models: ModelGroup[],
|
||||
deployments: AutoRouterDeployment[],
|
||||
costMap: ModelCostMap,
|
||||
): ComplexityRouterConfigValue | null => {
|
||||
const names = Array.from(
|
||||
new Set(
|
||||
models
|
||||
.filter((model) => model.mode === undefined || model.mode === "chat")
|
||||
.map((model) => model.model_group)
|
||||
.filter((name) => name && !name.startsWith("auto_router/")),
|
||||
),
|
||||
);
|
||||
if (names.length === 0) return null;
|
||||
|
||||
const ranked = names
|
||||
.map((name) => ({ name, price: groupPrice(name, deployments, costMap) }))
|
||||
.sort((left, right) => {
|
||||
if (left.price === undefined && right.price !== undefined) return 1;
|
||||
if (left.price !== undefined && right.price === undefined) return -1;
|
||||
if (left.price !== undefined && right.price !== undefined && left.price !== right.price) {
|
||||
return left.price - right.price;
|
||||
}
|
||||
return left.name.localeCompare(right.name);
|
||||
})
|
||||
.map(({ name }) => name);
|
||||
|
||||
let selected: [string, string, string, string];
|
||||
if (ranked.length === 1) selected = [ranked[0], ranked[0], ranked[0], ranked[0]];
|
||||
else if (ranked.length === 2) selected = [ranked[0], ranked[0], ranked[1], ranked[1]];
|
||||
else if (ranked.length === 3) selected = [ranked[0], ranked[1], ranked[2], ranked[2]];
|
||||
else {
|
||||
const last = ranked.length - 1;
|
||||
selected = [ranked[0], ranked[Math.floor(last / 3)], ranked[Math.floor((2 * last) / 3)], ranked[last]];
|
||||
}
|
||||
|
||||
return {
|
||||
tiers: {
|
||||
SIMPLE: [selected[0]],
|
||||
MEDIUM: [selected[1]],
|
||||
COMPLEX: [selected[2]],
|
||||
REASONING: [selected[3]],
|
||||
},
|
||||
classifier_type: "heuristic_v2",
|
||||
};
|
||||
};
|
||||
Loading…
Add table
Reference in a new issue