fix(ui): refresh Auto Setup model ladders

This commit is contained in:
moe-berri 2026-09-03 19:43:55 -07:00
parent 78c40ed6a7
commit 2cb27985d9
3 changed files with 95 additions and 13 deletions

View file

@ -273,8 +273,8 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
[presets, availability],
);
const automaticRouterConfig = React.useMemo(
() => buildAutomaticRouterConfig(modelInfo, deployments ?? [], preferredTierModels),
[modelInfo, deployments, preferredTierModels],
() => buildAutomaticRouterConfig(modelInfo, deployments ?? [], preferredTierModels, availability),
[modelInfo, deployments, preferredTierModels, availability],
);
// A preset's models can only be trusted against a successfully loaded list. Selection and the

View file

@ -13,18 +13,38 @@ const tierModels = (config: ReturnType<typeof buildAutomaticRouterConfig>) =>
describe("buildPreferredTierModels", () => {
it("recognizes curated models that are not in a preset", () => {
const available = ["gpt-4o-mini", "claude-sonnet-4-5", "grok-4", "deepseek-reasoner"];
const available = ["gpt-5.6-luna", "claude-sonnet-5", "grok-4.6", "deepseek-v4-pro"];
const availability = buildModelAvailability(available, []);
const preferred = buildPreferredTierModels([], availability);
const expected: PreferredTierModels = {
SIMPLE: ["gpt-4o-mini"],
MEDIUM: ["claude-sonnet-4-5"],
COMPLEX: ["grok-4"],
REASONING: ["deepseek-reasoner"],
SIMPLE: ["gpt-5.6-luna"],
MEDIUM: ["claude-sonnet-5"],
COMPLEX: ["deepseek-v4-pro", "grok-4.6"],
REASONING: ["deepseek-v4-pro", "grok-4.6"],
};
expect(preferred).toEqual(expected);
});
it("prefers the current model ladder over older preset entries", () => {
const availability = buildModelAvailability(["gpt-5.6-luna", "gpt-4o-mini"], []);
const preferred = buildPreferredTierModels(
[
{
key: "old",
label: "Old",
description: "Old model",
complexity_router_config: {
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
classifier_type: "heuristic_v2",
},
},
],
availability,
);
expect(preferred.SIMPLE).toEqual(["gpt-5.6-luna", "gpt-4o-mini"]);
});
});
describe("buildAutomaticRouterConfig", () => {
@ -41,6 +61,49 @@ describe("buildAutomaticRouterConfig", () => {
).toEqual(["simple", "medium", "complex", "reasoning"]);
});
it.each([
{
provider: "OpenAI",
available: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra"],
expected: ["gpt-5.6-luna", "gpt-5.6-terra", "gpt-6-astra", "gpt-6-astra"],
effort: "max",
},
{
provider: "Anthropic",
available: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5"],
expected: ["claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5", "claude-opus-5"],
effort: "max",
},
{
provider: "Google",
available: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview"],
expected: ["gemini-3.5-flash-lite", "gemini-3.8-flash", "gemini-3.1-pro-preview", "gemini-3.1-pro-preview"],
effort: "high",
},
{
provider: "DeepSeek",
available: ["deepseek-v4-flash", "deepseek-v4-pro"],
expected: ["deepseek-v4-flash", "deepseek-v4-flash", "deepseek-v4-pro", "deepseek-v4-pro"],
effort: "high",
},
{
provider: "xAI",
available: ["grok-4.6"],
expected: ["grok-4.6", "grok-4.6", "grok-4.6", "grok-4.6"],
effort: "xhigh",
},
])("uses the current $provider ladder and maximum reasoning effort", ({ available, expected, effort }) => {
const availability = buildModelAvailability(available, []);
const preferred = buildPreferredTierModels([], availability);
const config = buildAutomaticRouterConfig(models(...available), [], preferred, availability);
expect(tierModels(config)).toEqual(expected);
expect(config?.tier_model_params).toEqual({
REASONING: { [expected[3]]: { reasoning_effort: effort } },
});
});
it("reuses the closest available tier when a tier has no match", () => {
const preferred: PreferredTierModels = {
SIMPLE: ["simple"],

View file

@ -7,11 +7,20 @@ const TIER_NAMES = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"] as const;
type TierName = (typeof TIER_NAMES)[number];
export type PreferredTierModels = Record<TierName, string[]>;
const ADDITIONAL_TIER_MODELS: PreferredTierModels = {
SIMPLE: ["gpt-4o-mini", "gpt-5-mini", "gemini-2.5-flash", "deepseek-chat"],
MEDIUM: ["gpt-5-mini", "gpt-4o", "claude-sonnet-4-5", "gemini-2.5-flash", "deepseek-chat"],
COMPLEX: ["gpt-5", "gpt-4o", "claude-sonnet-4-6", "gemini-2.5-pro", "grok-4"],
REASONING: ["o3", "deepseek-reasoner", "claude-opus-4-6", "gemini-2.5-pro", "gpt-5"],
const CURRENT_TIER_MODELS: PreferredTierModels = {
SIMPLE: ["gpt-5.6-luna", "claude-haiku-4-5", "gemini-3.5-flash-lite", "deepseek-v4-flash"],
MEDIUM: ["gpt-5.6-terra", "claude-sonnet-5", "gemini-3.8-flash", "deepseek-v4-flash"],
COMPLEX: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"],
REASONING: ["gpt-6-astra", "gpt-5.6-sol", "claude-opus-5", "gemini-3.1-pro-preview", "deepseek-v4-pro", "grok-4.6"],
};
const MAX_REASONING_EFFORT: Record<string, string> = {
"gpt-6-astra": "max",
"gpt-5.6-sol": "max",
"claude-opus-5": "max",
"gemini-3.1-pro-preview": "high",
"deepseek-v4-pro": "high",
"grok-4.6": "xhigh",
};
export const buildPreferredTierModels = (
@ -24,8 +33,8 @@ export const buildPreferredTierModels = (
Array.from(
new Set(
[
...CURRENT_TIER_MODELS[tier],
...presets.flatMap((preset) => preset.complexity_router_config.tiers[tier]),
...ADDITIONAL_TIER_MODELS[tier],
].flatMap((model) => {
const resolved = resolveAvailableModel(model, availability);
return resolved ? [resolved] : [];
@ -54,6 +63,7 @@ export const buildAutomaticRouterConfig = (
models: ModelGroup[],
deployments: AutoRouterDeployment[],
preferredByTier: PreferredTierModels,
availability?: ModelAvailability,
): ComplexityRouterConfigValue | null => {
const autoRouterNames: ReadonlySet<string> = new Set(
deployments
@ -73,6 +83,12 @@ export const buildAutomaticRouterConfig = (
const selected = selectPreferredTierModels(preferredByTier, usableNames);
if (selected === null) return null;
const reasoningEffort =
availability &&
Object.entries(MAX_REASONING_EFFORT).find(
([model]) => resolveAvailableModel(model, availability) === selected[3],
)?.[1];
return {
tiers: {
SIMPLE: [selected[0]],
@ -81,5 +97,8 @@ export const buildAutomaticRouterConfig = (
REASONING: [selected[3]],
},
classifier_type: "heuristic_v2",
...(reasoningEffort && {
tier_model_params: { REASONING: { [selected[3]]: { reasoning_effort: reasoningEffort } } },
}),
};
};