fix(UI): probe the pinned default model in the auto router connection test

The connection test built its targets from the tiers and the embedding model
only, so a Default Model pin outside every tier was never reached and a green
result could hide an unreachable default. model_info_view had already hand
rolled the dedupe and append locally, so the rule moved into
buildAutoRouterTestTargets and both call sites now share it.
This commit is contained in:
Tin Chi Lo 2026-08-13 12:26:57 -07:00
parent b6f9169c47
commit de2c59addc
4 changed files with 77 additions and 21 deletions

View file

@ -395,11 +395,13 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
};
const handleTestConnection = () => {
const targets = buildAutoRouterTestTargets({
const testTargetParams = {
tiers: complexityRouterConfig.tiers,
semanticMatchingEnabled,
embeddingModel,
});
defaultModel: resolveComplexityDefaultModel(complexityRouterConfig.tiers, complexityRouterConfig.default_model),
};
const targets = buildAutoRouterTestTargets(testTargetParams);
if (targets.length === 0) {
NotificationManager.fromBackend("Please select at least one model for a complexity tier");

View file

@ -77,4 +77,46 @@ describe("buildAutoRouterTestTargets", () => {
});
expect(targets).toEqual([{ labels: ["SIMPLE"], modelGroup: "gpt-4o-mini", mode: "chat" }]);
});
// A pin outside every tier is still a live fallback destination (empty-tier landings, and a
// failed LLM classifier routing to it), so a passing test must reach it too or it is only
// proving the tiers are reachable, not the router.
it("appends the default model as its own target when it is not already in a tier", () => {
const targets = buildAutoRouterTestTargets({
tiers,
semanticMatchingEnabled: false,
embeddingModel: undefined,
defaultModel: "claude-3-opus",
});
expect(targets).toEqual([
{ labels: ["SIMPLE"], modelGroup: "gpt-4o-mini", mode: "chat" },
{ labels: ["MEDIUM", "COMPLEX"], modelGroup: "claude-sonnet-4", mode: "chat" },
{ labels: ["REASONING"], modelGroup: "o3", mode: "chat" },
{ labels: ["Default"], modelGroup: "claude-3-opus", mode: "chat" },
]);
});
it("does not duplicate a default model that a tier already covers", () => {
const targets = buildAutoRouterTestTargets({
tiers,
semanticMatchingEnabled: false,
embeddingModel: undefined,
defaultModel: "claude-sonnet-4",
});
expect(targets).toEqual([
{ labels: ["SIMPLE"], modelGroup: "gpt-4o-mini", mode: "chat" },
{ labels: ["MEDIUM", "COMPLEX"], modelGroup: "claude-sonnet-4", mode: "chat" },
{ labels: ["REASONING"], modelGroup: "o3", mode: "chat" },
]);
});
it.each([[undefined], [""], [" "]])("adds no default target for %o", (defaultModel) => {
const targets = buildAutoRouterTestTargets({
tiers: { SIMPLE: ["gpt-4o-mini"], MEDIUM: [], COMPLEX: [], REASONING: [] },
semanticMatchingEnabled: false,
embeddingModel: undefined,
defaultModel,
});
expect(targets).toEqual([{ labels: ["SIMPLE"], modelGroup: "gpt-4o-mini", mode: "chat" }]);
});
});

View file

@ -12,6 +12,9 @@ export interface BuildAutoRouterTestTargetsParams {
tiers: ComplexityTiers;
semanticMatchingEnabled: boolean;
embeddingModel: string | undefined;
/** The resolved default model - see resolveComplexityDefaultModel. A live fallback destination,
* so it is probed even when no tier lists it. */
defaultModel?: string;
}
// Keys drive iteration order; `satisfies Record<keyof ComplexityTiers, null>` makes it a
@ -27,8 +30,9 @@ export const buildAutoRouterTestTargets = ({
tiers,
semanticMatchingEnabled,
embeddingModel,
defaultModel,
}: BuildAutoRouterTestTargetsParams): AutoRouterTestTarget[] => {
const groupedByModel = TIER_ORDER.reduce<Record<string, string[]>>((acc, tier) => {
const tieredByModel = TIER_ORDER.reduce<Record<string, string[]>>((acc, tier) => {
return (tiers[tier] ?? []).reduce((tierAcc, rawModel) => {
const modelGroup = rawModel?.trim();
if (!modelGroup) return tierAcc;
@ -36,6 +40,15 @@ export const buildAutoRouterTestTargets = ({
}, acc);
}, {});
// The default is a live destination whenever the chosen tier has no model, and when an LLM
// classifier fails with "Route to the default model", so a green test that skipped it would be
// reporting on a router it had not fully reached.
const resolvedDefault = defaultModel?.trim();
const groupedByModel =
resolvedDefault && !(resolvedDefault in tieredByModel)
? { ...tieredByModel, [resolvedDefault]: ["Default"] }
: tieredByModel;
const tierTargets: AutoRouterTestTarget[] = Object.entries(groupedByModel).map(([modelGroup, labels]) => ({
labels,
modelGroup,

View file

@ -40,7 +40,7 @@ import { isMaskedSecret, stripMaskedSecrets } from "../utils/maskedSecretUtils";
import { formItemValidateJSON, truncateString } from "../utils/textUtils";
import AutoRouterConnectionTest from "./add_model/auto_router_connection_test";
import { AutoRouterTestTarget, buildAutoRouterTestTargets } from "./add_model/build_auto_router_test_targets";
import { normalizeTierModels } from "./add_model/complexity_router_tiers";
import { normalizeTierModels, resolveComplexityDefaultModel } from "./add_model/complexity_router_tiers";
import {
hasAutoRouterEditor,
isAutoRouterDeployment,
@ -146,6 +146,7 @@ interface ComplexityRouterTierConfig {
};
semantic_keyword_matching?: boolean;
embedding_model?: string;
default_model?: string;
}
interface ComplexityRouterModelData {
@ -170,25 +171,23 @@ const buildComplexityRouterTestTargets = (
config = rawConfig;
}
const tierTargets = buildAutoRouterTestTargets({
tiers: {
SIMPLE: normalizeTierModels(config.tiers?.SIMPLE),
MEDIUM: normalizeTierModels(config.tiers?.MEDIUM),
COMPLEX: normalizeTierModels(config.tiers?.COMPLEX),
REASONING: normalizeTierModels(config.tiers?.REASONING),
},
const tiers = {
SIMPLE: normalizeTierModels(config.tiers?.SIMPLE),
MEDIUM: normalizeTierModels(config.tiers?.MEDIUM),
COMPLEX: normalizeTierModels(config.tiers?.COMPLEX),
REASONING: normalizeTierModels(config.tiers?.REASONING),
};
// litellm_params overrides the config blob at init (complexity_router.py), so it wins here too.
const pinned = modelData?.litellm_params?.complexity_router_default_model || config.default_model;
const testTargetParams = {
tiers,
semanticMatchingEnabled: Boolean(config.semantic_keyword_matching),
embeddingModel: config.embedding_model,
});
const defaultModel = modelData?.litellm_params?.complexity_router_default_model?.trim();
if (!defaultModel || tierTargets.some((target) => target.modelGroup === defaultModel)) {
return tierTargets;
}
return [
...tierTargets,
{ labels: ["Default (unconfigured tiers)"], modelGroup: defaultModel, mode: "chat" as const },
];
defaultModel: resolveComplexityDefaultModel(tiers, pinned),
};
return buildAutoRouterTestTargets(testTargetParams);
};
export default function ModelInfoView({