feat(ui): prefer proven models in Auto Setup fallback

This commit is contained in:
moe-berri 2026-09-03 19:00:26 -07:00
parent a48afd2242
commit 72ddd699dd
5 changed files with 136 additions and 8 deletions

View file

@ -210,6 +210,25 @@ describe("AddAutoRouterTab", () => {
expect(toast.success).toHaveBeenCalledWith(`Configured with ${openAiPreset.label}`);
});
it("mixes available models from the preferred tier catalog when no complete template fits", async () => {
mockFetchAvailableModels.mockResolvedValue(
["gpt-5.6-luna", "claude-sonnet-5", "gpt-5.6-sol"].map((model_group) => ({
model_group,
mode: "chat",
})),
);
mockFetchAllModelDeployments.mockResolvedValue([]);
renderWithProviders(<Harness />);
const button = screen.getByTestId("configure-automatically-button");
await waitFor(() => expect(button).toBeEnabled());
await userEvent.click(button);
expect(
screen.getByText(/Simple: gpt-5.6-luna.*Medium: claude-sonnet-5.*Complex: gpt-5.6-sol.*Reasoning: gpt-5.6-sol/),
).toBeInTheDocument();
});
// Nothing is filled in, so there is nothing to submit. The button reports that itself instead of
// accepting a click and answering with a toast.
it("offers no submit at all until every tier has a model", async () => {

View file

@ -65,7 +65,7 @@ import {
} from "@/lib/autorouter_presets";
import { useAutoRouterPresets } from "@/app/(dashboard)/hooks/autoRouter/useAutoRouterPresets";
import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
import { buildAutomaticRouterConfig } from "./auto_setup";
import { buildAutomaticRouterConfig, buildPreferredTierModels } from "./auto_setup";
interface AddAutoRouterTabProps {
handleOk: () => void;
@ -319,9 +319,11 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
};
const handleAutomaticSetup = () => {
const matchingPreset = AUTO_SETUP_PRESET_PRIORITY.map((key) => presets.find((preset) => preset.key === key)).find(
(preset) => preset && presetAvailability(preset).kind === "available",
);
const prioritizedPresets = AUTO_SETUP_PRESET_PRIORITY.flatMap((key) => {
const preset = presets.find((candidate) => candidate.key === key);
return preset ? [preset] : [];
});
const matchingPreset = prioritizedPresets.find((preset) => presetAvailability(preset).kind === "available");
if (matchingPreset) {
const presetState = presetAvailability(matchingPreset);
setSelectedPreset(matchingPreset.key);
@ -331,7 +333,8 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
return;
}
const generatedConfig = buildAutomaticRouterConfig(modelInfo, deployments ?? [], modelCostMap);
const preferredTierModels = buildPreferredTierModels(prioritizedPresets, availability);
const generatedConfig = buildAutomaticRouterConfig(modelInfo, deployments ?? [], modelCostMap, preferredTierModels);
if (generatedConfig === null) {
toast.fromError("Add at least one chat model before configuring an Auto Router");
return;

View file

@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";
import type { AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
import { buildAutomaticRouterConfig } from "./auto_setup";
import { buildAutomaticRouterConfig, type PreferredTierModels } from "./auto_setup";
const models = (...names: string[]) => names.map((model_group) => ({ model_group, mode: "chat" }));
@ -24,6 +24,68 @@ const tierModels = (config: ReturnType<typeof buildAutomaticRouterConfig>) =>
];
describe("buildAutomaticRouterConfig", () => {
it("uses available preferred models before price ranking", () => {
const preferred: PreferredTierModels = {
SIMPLE: ["preferred-simple"],
MEDIUM: ["preferred-medium"],
COMPLEX: ["preferred-complex"],
REASONING: ["preferred-reasoning"],
};
const available = [...Object.values(preferred).flat(), "cheap-decoy", "expensive-decoy"];
const config = buildAutomaticRouterConfig(
models(...available),
available.map((name, index) => deployment(name, index + 1)),
{},
preferred,
);
expect(tierModels(config)).toEqual([
"preferred-simple",
"preferred-medium",
"preferred-complex",
"preferred-reasoning",
]);
});
it("reuses the nearest preferred model for tiers with no preferred match", () => {
const preferred: PreferredTierModels = {
SIMPLE: ["preferred-simple"],
MEDIUM: [],
COMPLEX: ["preferred-complex"],
REASONING: [],
};
const config = buildAutomaticRouterConfig(
models("preferred-simple", "preferred-complex", "cheap-decoy"),
[deployment("preferred-simple", 4), deployment("preferred-complex", 5), deployment("cheap-decoy", 1)],
{},
preferred,
);
expect(tierModels(config)).toEqual([
"preferred-simple",
"preferred-simple",
"preferred-complex",
"preferred-complex",
]);
});
it("uses price ranking when none of the preferred models are available", () => {
const unavailablePreferred: PreferredTierModels = {
SIMPLE: ["missing-simple"],
MEDIUM: ["missing-medium"],
COMPLEX: ["missing-complex"],
REASONING: ["missing-reasoning"],
};
const config = buildAutomaticRouterConfig(
models("expensive", "cheap", "premium", "middle"),
[deployment("cheap", 1), deployment("middle", 2), deployment("premium", 3), deployment("expensive", 4)],
{},
unavailablePreferred,
);
expect(tierModels(config)).toEqual(["cheap", "middle", "premium", "expensive"]);
});
it("uses four different models when four are available", () => {
const config = buildAutomaticRouterConfig(
models("expensive", "cheap", "premium", "middle"),

View file

@ -1,5 +1,6 @@
import { isAutoRouterDeployment, type AutoRouterDeployment } from "@/app/(dashboard)/hooks/models/useModels";
import type { ModelGroup } from "@/components/llm_calls/fetch_models";
import { resolveAvailableModel, type AutoRouterPreset, type ModelAvailability } from "@/lib/autorouter_presets";
import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
type ModelCost = {
@ -9,6 +10,10 @@ type ModelCost = {
export type ModelCostMap = Record<string, ModelCost>;
const TIER_NAMES = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const;
type TierName = (typeof TIER_NAMES)[number];
export type PreferredTierModels = Record<TierName, string[]>;
const price = (cost: ModelCost | null | undefined): number | undefined => {
const input = cost?.input_cost_per_token;
const output = cost?.output_cost_per_token;
@ -56,10 +61,46 @@ const selectTierModels = (ranked: string[]): [string, string, string, string] =>
return [ranked[0], ranked[Math.floor(last / 3)], ranked[Math.floor((2 * last) / 3)], ranked[last]];
};
export const buildPreferredTierModels = (
presets: AutoRouterPreset[],
availability: ModelAvailability,
): PreferredTierModels =>
Object.fromEntries(
TIER_NAMES.map((tier) => [
tier,
Array.from(
new Set(
presets.flatMap((preset) =>
preset.complexity_router_config.tiers[tier].flatMap((model) => {
const resolved = resolveAvailableModel(model, availability);
return resolved ? [resolved] : [];
}),
),
),
),
]),
) as PreferredTierModels;
const selectPreferredTierModels = (
preferredByTier: PreferredTierModels,
usableNames: ReadonlySet<string>,
): [string, string, string, string] | null => {
const preferred = TIER_NAMES.map((tier) => preferredByTier[tier].find((name) => usableNames.has(name)));
const candidates = preferred.flatMap((model, tier) => (model ? [{ model, tier }] : []));
if (candidates.length === 0) return null;
const nearest = (tier: number): string =>
[...candidates].sort(
(left, right) => Math.abs(left.tier - tier) - Math.abs(right.tier - tier) || left.tier - right.tier,
)[0].model;
return preferred.map((model, tier) => model ?? nearest(tier)) as [string, string, string, string];
};
export const buildAutomaticRouterConfig = (
models: ModelGroup[],
deployments: AutoRouterDeployment[],
costMap: ModelCostMap,
preferredByTier?: PreferredTierModels,
): ComplexityRouterConfigValue | null => {
const autoRouterNames: ReadonlySet<string> = new Set(
deployments
@ -75,6 +116,7 @@ export const buildAutomaticRouterConfig = (
),
);
if (names.length === 0) return null;
const usableNames: ReadonlySet<string> = new Set(names);
const ranked = names
.map((name) => ({ name, price: groupPrice(name, deployments, costMap) }))
@ -88,7 +130,9 @@ export const buildAutomaticRouterConfig = (
})
.map(({ name }) => name);
const selected = selectTierModels(ranked);
const selected = preferredByTier
? selectPreferredTierModels(preferredByTier, usableNames) ?? selectTierModels(ranked)
: selectTierModels(ranked);
return {
tiers: {

View file

@ -152,7 +152,7 @@ export const deploymentRefsFromModelInfo = (
return row.model_name && underlyingModels.length > 0 ? [{ modelGroup: row.model_name, underlyingModels }] : [];
});
const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => {
export const resolveAvailableModel = (requiredModel: string, availability: ModelAvailability): string | undefined => {
const { modelGroups, underlyingIndex } = availability;
if (modelGroups.has(requiredModel)) return requiredModel;
const normalized = normalizeModelName(requiredModel);