mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
test(ui): catch a preset spelling the same model two ways in one tier
buildPresetPrefill resolves every model reference through normalizeModelName, so two spellings of the same model in one tier (e.g. "claude-sonnet-4-5" and "claude-sonnet-4.5") collapse to one key. For tier_model_configs that means one model's litellm_params silently overwrites the other's - flagged by Greptile on #38453 (P2, confirmed real via a throwaway repro, not a regression: on the merge base both param sets were already dropped). Nothing else validates preset authoring, and these are trusted, checked-in JSON, so the fix is a static test over the bundled data rather than runtime code. Exports normalizeModelName so the test exercises the actual resolution rule instead of a hand-rolled copy of it. Verified the test fails when a preset is mutated to spell one model two ways, and passes clean on the real bundled presets.
This commit is contained in:
parent
dc25608ea1
commit
7174fa7a4c
2 changed files with 25 additions and 1 deletions
|
|
@ -11,6 +11,7 @@ import {
|
||||||
buildPresetPrefill,
|
buildPresetPrefill,
|
||||||
buildModelAvailability,
|
buildModelAvailability,
|
||||||
deploymentRefsFromModelInfo,
|
deploymentRefsFromModelInfo,
|
||||||
|
normalizeModelName,
|
||||||
} from "./autorouter_presets";
|
} from "./autorouter_presets";
|
||||||
import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching";
|
import { DEFAULT_MATCH_THRESHOLD } from "@/components/add_model/SemanticKeywordMatching";
|
||||||
import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords";
|
import { DEFAULT_ESCALATION_KEYWORDS } from "@/components/add_model/EscalationKeywords";
|
||||||
|
|
@ -28,6 +29,29 @@ describe("autorouter_presets", () => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// buildPresetPrefill resolves every model reference through normalizeModelName, so two spellings
|
||||||
|
// of the same model in one tier (e.g. "claude-sonnet-4-5" and "claude-sonnet-4.5") collapse to one
|
||||||
|
// key. For tier_model_configs that silently drops one model's litellm_params; catch it in the
|
||||||
|
// bundled data itself, since nothing else validates preset authoring.
|
||||||
|
it("never spells the same model two ways within a single tier", () => {
|
||||||
|
for (const preset of getAllPresets()) {
|
||||||
|
const { tiers, tier_model_configs: configs } = preset.complexity_router_config;
|
||||||
|
for (const tier of Object.keys(tiers) as (keyof typeof tiers)[]) {
|
||||||
|
const fromTierList = tiers[tier] ?? [];
|
||||||
|
const fromConfigs = (configs?.[tier] ?? []).map((entry) => entry.model_name);
|
||||||
|
const names = new Set([...fromTierList, ...fromConfigs]);
|
||||||
|
const byNormalized = new Map<string, string[]>();
|
||||||
|
for (const name of names) {
|
||||||
|
const key = normalizeModelName(name);
|
||||||
|
byNormalized.set(key, [...(byNormalized.get(key) ?? []), name]);
|
||||||
|
}
|
||||||
|
for (const spellings of byNormalized.values()) {
|
||||||
|
expect(new Set(spellings).size, `${preset.key}.${tier}: ${spellings.join(", ")}`).toBe(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("resolves a preset by its stable JSON key, not its display label", () => {
|
it("resolves a preset by its stable JSON key, not its display label", () => {
|
||||||
expect(getPresetByKey("anthropic_family")?.label).toBe("Anthropic Family");
|
expect(getPresetByKey("anthropic_family")?.label).toBe("Anthropic Family");
|
||||||
expect(getPresetByKey("does_not_exist")).toBeUndefined();
|
expect(getPresetByKey("does_not_exist")).toBeUndefined();
|
||||||
|
|
|
||||||
|
|
@ -55,7 +55,7 @@ export const getRequiredModels = (
|
||||||
// differing only in that separator. Canonicalizing on "-" (the presets' own convention) lets both
|
// differing only in that separator. Canonicalizing on "-" (the presets' own convention) lets both
|
||||||
// spellings match without doing anything looser - two DIFFERENT model names never collide here,
|
// spellings match without doing anything looser - two DIFFERENT model names never collide here,
|
||||||
// only the punctuation within one version number does.
|
// only the punctuation within one version number does.
|
||||||
const normalizeModelName = (model: string): string => model.replace(/(\d)\.(\d)/g, "$1-$2");
|
export const normalizeModelName = (model: string): string => model.replace(/(\d)\.(\d)/g, "$1-$2");
|
||||||
|
|
||||||
export interface DeploymentModelRef {
|
export interface DeploymentModelRef {
|
||||||
modelGroup: string;
|
modelGroup: string;
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue