diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index a433808085b..0613b0c02ae 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -4,9 +4,10 @@ import { vi } from "vitest"; import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; const mockModelInfo = [ - { model_group: "gpt-4" }, - { model_group: "gpt-3.5-turbo" }, - { model_group: "claude-3-opus" }, + { model_group: "gpt-4", mode: "chat" }, + { model_group: "gpt-3.5-turbo", mode: "chat" }, + { model_group: "claude-3-opus", mode: "chat" }, + { model_group: "text-embedding-3-small", mode: "embedding" }, ] as any[]; const defaultValue: ComplexityRouterConfigValue = { @@ -207,4 +208,32 @@ describe("ComplexityRouterConfig", () => { await user.click(screen.getByRole("switch")); expect(onSemanticMatchingEnabledChange).toHaveBeenCalledWith(true, expect.anything()); }); + + it("excludes embedding-mode models from the tier and classifier dropdowns", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const simpleTierSection = screen.getByText("Simple Tier").closest(".mb-4") as HTMLElement; + const combobox = within(simpleTierSection).getByRole("combobox"); + await user.click(combobox); + + expect((await screen.findAllByText("gpt-3.5-turbo")).length).toBeGreaterThan(0); + expect(screen.queryAllByText("text-embedding-3-small")).toHaveLength(0); + }); + + it("does not show tier validation errors by default", () => { + renderWithProviders(); + expect(screen.queryByText("This tier is required")).not.toBeInTheDocument(); + }); + + it("shows a validation error only under unfilled tiers when showValidationErrors is true", () => { + renderWithProviders( + , + ); + expect(screen.getAllByText("This tier is required")).toHaveLength(1); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 555648db8ad..18c575c7c4c 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -45,6 +45,7 @@ interface ComplexityRouterConfigProps { onEmbeddingModelChange?: (model: string) => void; matchThreshold?: number; onMatchThresholdChange?: (threshold: number) => void; + showValidationErrors?: boolean; } const TIER_DESCRIPTIONS: Record = { @@ -84,12 +85,15 @@ const ComplexityRouterConfig: React.FC = ({ onEmbeddingModelChange = () => {}, matchThreshold = 0.5, onMatchThresholdChange = () => {}, + showValidationErrors = false, }) => { - // Prepare model options for dropdowns - const modelOptions = modelInfo.map((model) => ({ - value: model.model_group, - label: model.model_group, - })); + // Embedding models can't serve a chat-completion role, so they're excluded here. + const modelOptions = modelInfo + .filter((model) => model.mode !== "embedding") + .map((model) => ({ + value: model.model_group, + label: model.model_group, + })); const handleTierChange = (tier: keyof ComplexityTiers, model: string) => { onChange({ @@ -148,6 +152,7 @@ const ComplexityRouterConfig: React.FC = ({ {(Object.keys(TIER_DESCRIPTIONS) as Array).map((tier, index) => { const tierInfo = TIER_DESCRIPTIONS[tier]; + const tierMissing = showValidationErrors && !value.tiers[tier]; return (
{index > 0 && } @@ -170,7 +175,13 @@ const ComplexityRouterConfig: React.FC = ({ showSearch style={{ width: "100%" }} options={modelOptions} + status={tierMissing ? "error" : undefined} /> + {tierMissing && ( + + This tier is required + + )}
); @@ -323,6 +334,7 @@ const ComplexityRouterConfig: React.FC = ({ matchThreshold={matchThreshold} onMatchThresholdChange={onMatchThresholdChange} modelInfo={modelInfo} + showValidationErrors={showValidationErrors} /> )} diff --git a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.test.tsx b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.test.tsx new file mode 100644 index 00000000000..2336e6faf43 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.test.tsx @@ -0,0 +1,53 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import SemanticKeywordMatching from "./SemanticKeywordMatching"; + +const mockModelInfo = [ + { model_group: "gpt-4", mode: "chat" }, + { model_group: "text-embedding-3-small", mode: "embedding" }, + { model_group: "voyage-3-5", mode: "embedding" }, + { model_group: "legacy-model" }, +] as any[]; + +const baseProps = { + enabled: true, + onEnabledChange: vi.fn(), + embeddingModel: undefined, + onEmbeddingModelChange: vi.fn(), + matchThreshold: 0.5, + onMatchThresholdChange: vi.fn(), + modelInfo: mockModelInfo, +}; + +describe("SemanticKeywordMatching", () => { + it("only lists embedding-mode models in the embedding model dropdown", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await user.click(combobox); + + expect((await screen.findAllByText("text-embedding-3-small")).length).toBeGreaterThan(0); + expect(screen.getAllByText("voyage-3-5").length).toBeGreaterThan(0); + expect(screen.queryAllByText("gpt-4")).toHaveLength(0); + expect(screen.queryAllByText("legacy-model")).toHaveLength(0); + }); + + it("does not show a validation error by default", () => { + renderWithProviders(); + expect(screen.queryByText("An embedding model is required")).not.toBeInTheDocument(); + }); + + it("shows a validation error when showValidationErrors is true and no embedding model is set", () => { + renderWithProviders(); + expect(screen.getByText("An embedding model is required")).toBeInTheDocument(); + }); + + it("hides the validation error once an embedding model is set", () => { + renderWithProviders( + , + ); + expect(screen.queryByText("An embedding model is required")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx index 0f9907ac6c9..c7583427af6 100644 --- a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx +++ b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx @@ -15,6 +15,7 @@ interface SemanticKeywordMatchingProps { matchThreshold: number; onMatchThresholdChange: (threshold: number) => void; modelInfo: ModelGroup[]; + showValidationErrors?: boolean; } const SemanticKeywordMatching: React.FC = ({ @@ -25,11 +26,14 @@ const SemanticKeywordMatching: React.FC = ({ matchThreshold, onMatchThresholdChange, modelInfo, + showValidationErrors = false, }) => { - const modelOptions = Array.from(new Set(modelInfo.map((model) => model.model_group))).map((model_group) => ({ + const embeddingModels = modelInfo.filter((model) => model.mode === "embedding"); + const modelOptions = Array.from(new Set(embeddingModels.map((model) => model.model_group))).map((model_group) => ({ value: model_group, label: model_group, })); + const embeddingModelMissing = showValidationErrors && !embeddingModel; return ( @@ -60,7 +64,13 @@ const SemanticKeywordMatching: React.FC = ({ showSearch style={{ width: "100%" }} options={modelOptions} + status={embeddingModelMissing ? "error" : undefined} /> + {embeddingModelMissing && ( + + An embedding model is required + + )}
Minimum match score diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 55c62224a48..ce69f8f7ae3 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -11,7 +11,11 @@ import RouterConfigBuilder from "./RouterConfigBuilder"; import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; import { KeywordTierRule } from "./KeywordTierRules"; import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching"; -import { buildComplexityRouterConfig, getSemanticConfigError } from "./build_complexity_router_config"; +import { + buildComplexityRouterConfig, + getMissingTiersError, + getSemanticConfigError, +} from "./build_complexity_router_config"; import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets"; import AutoRouterConnectionTest from "./auto_router_connection_test"; import NotificationManager from "../molecules/notifications_manager"; @@ -43,6 +47,7 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc const [semanticMatchingEnabled, setSemanticMatchingEnabled] = useState(false); const [embeddingModel, setEmbeddingModel] = useState(undefined); const [matchThreshold, setMatchThreshold] = useState(DEFAULT_MATCH_THRESHOLD); + const [showValidationErrors, setShowValidationErrors] = useState(false); // Semantic router config (existing) const [routerConfig, setRouterConfig] = useState(null); @@ -86,19 +91,22 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc classifier_llm_config: classifierLlmConfig, } = complexityRouterConfig; - const filledTiers = Object.values(tiers).filter(Boolean); - if (filledTiers.length === 0) { - NotificationManager.fromBackend("Please select at least one model for a complexity tier"); + const missingTiersError = getMissingTiersError(tiers); + if (missingTiersError) { + setShowValidationErrors(true); + NotificationManager.fromBackend(missingTiersError); return; } if (classifierType === "llm" && !classifierLlmConfig?.model) { + setShowValidationErrors(true); NotificationManager.fromBackend("Please select a classifier model, or switch back to Heuristic"); return; } const semanticError = getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules }); if (semanticError) { + setShowValidationErrors(true); NotificationManager.fromBackend(semanticError); return; } @@ -296,6 +304,7 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc onEmbeddingModelChange={setEmbeddingModel} matchThreshold={matchThreshold} onMatchThresholdChange={setMatchThreshold} + showValidationErrors={showValidationErrors} />
) : ( diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 3c252646b57..e5b547d8240 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -1,5 +1,6 @@ import { buildComplexityRouterConfig, + getMissingTiersError, getSemanticConfigError, BuildComplexityRouterConfigParams, } from "./build_complexity_router_config"; @@ -124,6 +125,31 @@ describe("buildComplexityRouterConfig", () => { }); }); +describe("getMissingTiersError", () => { + it("returns null when all four tiers have a model", () => { + expect(getMissingTiersError(tiers)).toBeNull(); + }); + + it("names the specific missing tier when only one is blank", () => { + expect(getMissingTiersError({ ...tiers, REASONING: "" })).toBe( + "Select a model for the following tier(s): REASONING", + ); + }); + + it("names multiple missing tiers in SIMPLE/MEDIUM/COMPLEX/REASONING order", () => { + expect(getMissingTiersError({ ...tiers, SIMPLE: "", REASONING: "" })).toBe( + "Select a model for the following tier(s): SIMPLE, REASONING", + ); + }); + + it("names all four tiers when none are filled", () => { + const noTiers = { SIMPLE: "", MEDIUM: "", COMPLEX: "", REASONING: "" }; + expect(getMissingTiersError(noTiers)).toBe( + "Select a model for the following tier(s): SIMPLE, MEDIUM, COMPLEX, REASONING", + ); + }); +}); + describe("getSemanticConfigError", () => { const rule = { id: "r1", keywords: ["k8s"], tier: "REASONING" as const }; diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 82ea4f8c12f..3eddca8c35b 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -30,6 +30,14 @@ export interface ComplexityRouterConfigPayload { match_threshold?: number; } +const TIER_KEYS: Array = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; + +export const getMissingTiersError = (tiers: ComplexityTiers): string | null => { + const missing = TIER_KEYS.filter((tier) => !tiers[tier]); + if (missing.length === 0) return null; + return `Select a model for the following tier(s): ${missing.join(", ")}`; +}; + export const getSemanticConfigError = ({ semanticMatchingEnabled, embeddingModel,