From 0de308a5b8f09af6d653bee4cdb9765cdf2c55ff Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:18:37 -0700 Subject: [PATCH 1/4] fix(auto_router): filter embedding models out of tier selects, require all tiers, add inline validation The Add Auto Router complexity tab let chat models fill the embedding-model slot (and vice versa) since neither dropdown filtered on ModelGroup.mode, and submit only required at least one of the four tiers instead of all four. Adds getMissingTiersError alongside the existing getSemanticConfigError, and highlights unfilled tier/embedding selects inline once a submit attempt fails. --- .../add_model/ComplexityRouterConfig.test.tsx | 35 ++++++++++-- .../add_model/ComplexityRouterConfig.tsx | 22 ++++++-- .../SemanticKeywordMatching.test.tsx | 53 +++++++++++++++++++ .../add_model/SemanticKeywordMatching.tsx | 12 ++++- .../add_model/add_auto_router_tab.tsx | 17 ++++-- .../build_complexity_router_config.test.ts | 26 +++++++++ .../build_complexity_router_config.ts | 8 +++ 7 files changed, 160 insertions(+), 13 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.test.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index a433808085b..0613b0c02ae 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -4,9 +4,10 @@ import { vi } from "vitest"; import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; const mockModelInfo = [ - { model_group: "gpt-4" }, - { model_group: "gpt-3.5-turbo" }, - { model_group: "claude-3-opus" }, + { model_group: "gpt-4", mode: "chat" }, + { model_group: "gpt-3.5-turbo", mode: "chat" }, + { model_group: "claude-3-opus", mode: "chat" }, + { model_group: "text-embedding-3-small", mode: "embedding" }, ] as any[]; const defaultValue: ComplexityRouterConfigValue = { @@ -207,4 +208,32 @@ describe("ComplexityRouterConfig", () => { await user.click(screen.getByRole("switch")); expect(onSemanticMatchingEnabledChange).toHaveBeenCalledWith(true, expect.anything()); }); + + it("excludes embedding-mode models from the tier and classifier dropdowns", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const simpleTierSection = screen.getByText("Simple Tier").closest(".mb-4") as HTMLElement; + const combobox = within(simpleTierSection).getByRole("combobox"); + await user.click(combobox); + + expect((await screen.findAllByText("gpt-3.5-turbo")).length).toBeGreaterThan(0); + expect(screen.queryAllByText("text-embedding-3-small")).toHaveLength(0); + }); + + it("does not show tier validation errors by default", () => { + renderWithProviders(); + expect(screen.queryByText("This tier is required")).not.toBeInTheDocument(); + }); + + it("shows a validation error only under unfilled tiers when showValidationErrors is true", () => { + renderWithProviders( + , + ); + expect(screen.getAllByText("This tier is required")).toHaveLength(1); + }); }); diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx index 555648db8ad..18c575c7c4c 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx @@ -45,6 +45,7 @@ interface ComplexityRouterConfigProps { onEmbeddingModelChange?: (model: string) => void; matchThreshold?: number; onMatchThresholdChange?: (threshold: number) => void; + showValidationErrors?: boolean; } const TIER_DESCRIPTIONS: Record = { @@ -84,12 +85,15 @@ const ComplexityRouterConfig: React.FC = ({ onEmbeddingModelChange = () => {}, matchThreshold = 0.5, onMatchThresholdChange = () => {}, + showValidationErrors = false, }) => { - // Prepare model options for dropdowns - const modelOptions = modelInfo.map((model) => ({ - value: model.model_group, - label: model.model_group, - })); + // Embedding models can't serve a chat-completion role, so they're excluded here. + const modelOptions = modelInfo + .filter((model) => model.mode !== "embedding") + .map((model) => ({ + value: model.model_group, + label: model.model_group, + })); const handleTierChange = (tier: keyof ComplexityTiers, model: string) => { onChange({ @@ -148,6 +152,7 @@ const ComplexityRouterConfig: React.FC = ({ {(Object.keys(TIER_DESCRIPTIONS) as Array).map((tier, index) => { const tierInfo = TIER_DESCRIPTIONS[tier]; + const tierMissing = showValidationErrors && !value.tiers[tier]; return (
{index > 0 && } @@ -170,7 +175,13 @@ const ComplexityRouterConfig: React.FC = ({ showSearch style={{ width: "100%" }} options={modelOptions} + status={tierMissing ? "error" : undefined} /> + {tierMissing && ( + + This tier is required + + )}
); @@ -323,6 +334,7 @@ const ComplexityRouterConfig: React.FC = ({ matchThreshold={matchThreshold} onMatchThresholdChange={onMatchThresholdChange} modelInfo={modelInfo} + showValidationErrors={showValidationErrors} /> )} diff --git a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.test.tsx b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.test.tsx new file mode 100644 index 00000000000..2336e6faf43 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.test.tsx @@ -0,0 +1,53 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import SemanticKeywordMatching from "./SemanticKeywordMatching"; + +const mockModelInfo = [ + { model_group: "gpt-4", mode: "chat" }, + { model_group: "text-embedding-3-small", mode: "embedding" }, + { model_group: "voyage-3-5", mode: "embedding" }, + { model_group: "legacy-model" }, +] as any[]; + +const baseProps = { + enabled: true, + onEnabledChange: vi.fn(), + embeddingModel: undefined, + onEmbeddingModelChange: vi.fn(), + matchThreshold: 0.5, + onMatchThresholdChange: vi.fn(), + modelInfo: mockModelInfo, +}; + +describe("SemanticKeywordMatching", () => { + it("only lists embedding-mode models in the embedding model dropdown", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + const combobox = screen.getByRole("combobox"); + await user.click(combobox); + + expect((await screen.findAllByText("text-embedding-3-small")).length).toBeGreaterThan(0); + expect(screen.getAllByText("voyage-3-5").length).toBeGreaterThan(0); + expect(screen.queryAllByText("gpt-4")).toHaveLength(0); + expect(screen.queryAllByText("legacy-model")).toHaveLength(0); + }); + + it("does not show a validation error by default", () => { + renderWithProviders(); + expect(screen.queryByText("An embedding model is required")).not.toBeInTheDocument(); + }); + + it("shows a validation error when showValidationErrors is true and no embedding model is set", () => { + renderWithProviders(); + expect(screen.getByText("An embedding model is required")).toBeInTheDocument(); + }); + + it("hides the validation error once an embedding model is set", () => { + renderWithProviders( + , + ); + expect(screen.queryByText("An embedding model is required")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx index 0f9907ac6c9..c7583427af6 100644 --- a/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx +++ b/ui/litellm-dashboard/src/components/add_model/SemanticKeywordMatching.tsx @@ -15,6 +15,7 @@ interface SemanticKeywordMatchingProps { matchThreshold: number; onMatchThresholdChange: (threshold: number) => void; modelInfo: ModelGroup[]; + showValidationErrors?: boolean; } const SemanticKeywordMatching: React.FC = ({ @@ -25,11 +26,14 @@ const SemanticKeywordMatching: React.FC = ({ matchThreshold, onMatchThresholdChange, modelInfo, + showValidationErrors = false, }) => { - const modelOptions = Array.from(new Set(modelInfo.map((model) => model.model_group))).map((model_group) => ({ + const embeddingModels = modelInfo.filter((model) => model.mode === "embedding"); + const modelOptions = Array.from(new Set(embeddingModels.map((model) => model.model_group))).map((model_group) => ({ value: model_group, label: model_group, })); + const embeddingModelMissing = showValidationErrors && !embeddingModel; return ( @@ -60,7 +64,13 @@ const SemanticKeywordMatching: React.FC = ({ showSearch style={{ width: "100%" }} options={modelOptions} + status={embeddingModelMissing ? "error" : undefined} /> + {embeddingModelMissing && ( + + An embedding model is required + + )}
Minimum match score diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 55c62224a48..ce69f8f7ae3 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -11,7 +11,11 @@ import RouterConfigBuilder from "./RouterConfigBuilder"; import ComplexityRouterConfig, { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; import { KeywordTierRule } from "./KeywordTierRules"; import { DEFAULT_MATCH_THRESHOLD } from "./SemanticKeywordMatching"; -import { buildComplexityRouterConfig, getSemanticConfigError } from "./build_complexity_router_config"; +import { + buildComplexityRouterConfig, + getMissingTiersError, + getSemanticConfigError, +} from "./build_complexity_router_config"; import { buildAutoRouterTestTargets, AutoRouterTestTarget } from "./build_auto_router_test_targets"; import AutoRouterConnectionTest from "./auto_router_connection_test"; import NotificationManager from "../molecules/notifications_manager"; @@ -43,6 +47,7 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc const [semanticMatchingEnabled, setSemanticMatchingEnabled] = useState(false); const [embeddingModel, setEmbeddingModel] = useState(undefined); const [matchThreshold, setMatchThreshold] = useState(DEFAULT_MATCH_THRESHOLD); + const [showValidationErrors, setShowValidationErrors] = useState(false); // Semantic router config (existing) const [routerConfig, setRouterConfig] = useState(null); @@ -86,19 +91,22 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc classifier_llm_config: classifierLlmConfig, } = complexityRouterConfig; - const filledTiers = Object.values(tiers).filter(Boolean); - if (filledTiers.length === 0) { - NotificationManager.fromBackend("Please select at least one model for a complexity tier"); + const missingTiersError = getMissingTiersError(tiers); + if (missingTiersError) { + setShowValidationErrors(true); + NotificationManager.fromBackend(missingTiersError); return; } if (classifierType === "llm" && !classifierLlmConfig?.model) { + setShowValidationErrors(true); NotificationManager.fromBackend("Please select a classifier model, or switch back to Heuristic"); return; } const semanticError = getSemanticConfigError({ semanticMatchingEnabled, embeddingModel, keywordTierRules }); if (semanticError) { + setShowValidationErrors(true); NotificationManager.fromBackend(semanticError); return; } @@ -296,6 +304,7 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc onEmbeddingModelChange={setEmbeddingModel} matchThreshold={matchThreshold} onMatchThresholdChange={setMatchThreshold} + showValidationErrors={showValidationErrors} />
) : ( diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts index 3c252646b57..e5b547d8240 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts @@ -1,5 +1,6 @@ import { buildComplexityRouterConfig, + getMissingTiersError, getSemanticConfigError, BuildComplexityRouterConfigParams, } from "./build_complexity_router_config"; @@ -124,6 +125,31 @@ describe("buildComplexityRouterConfig", () => { }); }); +describe("getMissingTiersError", () => { + it("returns null when all four tiers have a model", () => { + expect(getMissingTiersError(tiers)).toBeNull(); + }); + + it("names the specific missing tier when only one is blank", () => { + expect(getMissingTiersError({ ...tiers, REASONING: "" })).toBe( + "Select a model for the following tier(s): REASONING", + ); + }); + + it("names multiple missing tiers in SIMPLE/MEDIUM/COMPLEX/REASONING order", () => { + expect(getMissingTiersError({ ...tiers, SIMPLE: "", REASONING: "" })).toBe( + "Select a model for the following tier(s): SIMPLE, REASONING", + ); + }); + + it("names all four tiers when none are filled", () => { + const noTiers = { SIMPLE: "", MEDIUM: "", COMPLEX: "", REASONING: "" }; + expect(getMissingTiersError(noTiers)).toBe( + "Select a model for the following tier(s): SIMPLE, MEDIUM, COMPLEX, REASONING", + ); + }); +}); + describe("getSemanticConfigError", () => { const rule = { id: "r1", keywords: ["k8s"], tier: "REASONING" as const }; diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts index 82ea4f8c12f..3eddca8c35b 100644 --- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts +++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts @@ -30,6 +30,14 @@ export interface ComplexityRouterConfigPayload { match_threshold?: number; } +const TIER_KEYS: Array = ["SIMPLE", "MEDIUM", "COMPLEX", "REASONING"]; + +export const getMissingTiersError = (tiers: ComplexityTiers): string | null => { + const missing = TIER_KEYS.filter((tier) => !tiers[tier]); + if (missing.length === 0) return null; + return `Select a model for the following tier(s): ${missing.join(", ")}`; +}; + export const getSemanticConfigError = ({ semanticMatchingEnabled, embeddingModel, From 3464b5e7dfaf027e366f95f60c55e89265d6601e Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:03:48 -0700 Subject: [PATCH 2/4] fix(auto_router): reset inline validation errors when switching router type --- .../src/components/add_model/add_auto_router_tab.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index ce69f8f7ae3..4e7a72435bf 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -238,7 +238,14 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc
Router Type - setRouterType(e.target.value)} className="w-full"> + { + setRouterType(e.target.value); + setShowValidationErrors(false); + }} + className="w-full" + >
From d6883d15b0feac1bfe07eaa18414ef14b1a5a7f4 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:08:07 -0700 Subject: [PATCH 3/4] fix(auto_router): flag name field and tier fields together on empty submit Clicking Add Auto Router with the name empty returned early with only a toast, so blank tier selects never got their inline error state. The empty-name branch now sets showValidationErrors and triggers antd validation on the name field, so every unfilled mandatory field is flagged at once. Adds a regression test for the tab component. --- .../add_model/add_auto_router_tab.test.tsx | 40 +++++++++++++++++++ .../add_model/add_auto_router_tab.tsx | 2 + 2 files changed, 42 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx new file mode 100644 index 00000000000..4713f8c6869 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.test.tsx @@ -0,0 +1,40 @@ +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import userEvent from "@testing-library/user-event"; +import { vi } from "vitest"; +import { Form } from "antd"; +import AddAutoRouterTab from "./add_auto_router_tab"; +import NotificationManager from "../molecules/notifications_manager"; + +vi.mock("../networking", () => ({ + modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }), +})); + +vi.mock("@/components/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn().mockResolvedValue([]), +})); + +vi.mock("./handle_add_auto_router_submit", () => ({ + handleAddAutoRouterSubmit: vi.fn(), +})); + +vi.mock("../molecules/notifications_manager", () => ({ + default: { fromBackend: vi.fn() }, +})); + +const Harness = () => { + const [form] = Form.useForm(); + return ; +}; + +describe("AddAutoRouterTab", () => { + it("flags every mandatory field when Add Auto Router is clicked with nothing filled", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: /add auto router/i })); + + expect(await screen.findByText("Auto router name is required")).toBeInTheDocument(); + expect(screen.getAllByText("This tier is required")).toHaveLength(4); + expect(NotificationManager.fromBackend).toHaveBeenCalledWith("Please enter an Auto Router Name"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx index 4e7a72435bf..a74eab0abdd 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx @@ -198,6 +198,8 @@ const AddAutoRouterTab: React.FC = ({ form, handleOk, acc const handleAutoRouterSubmit = () => { const name = form.getFieldValue("auto_router_name"); if (!name) { + setShowValidationErrors(true); + form.validateFields(["auto_router_name"]).catch(() => undefined); NotificationManager.fromBackend("Please enter an Auto Router Name"); return; } From 0e90f61e48a84ae8fd7610333032a40b22e6c928 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:21:51 -0700 Subject: [PATCH 4/4] fix(auto_router): inline error for missing LLM classifier model Selecting the LLM classifier without picking a model only surfaced a toast on submit; the classifier model select now gets the same red outline and helper text as the tier and embedding selects once a submit attempt has failed. --- .../add_model/ComplexityRouterConfig.test.tsx | 22 +++++++++++++++++++ .../add_model/ComplexityRouterConfig.tsx | 9 ++++++++ 2 files changed, 31 insertions(+) diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx index 0613b0c02ae..a34a8709918 100644 --- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx @@ -226,6 +226,28 @@ describe("ComplexityRouterConfig", () => { expect(screen.queryByText("This tier is required")).not.toBeInTheDocument(); }); + it("shows an inline error on the classifier model select when llm is selected without a model", () => { + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "", timeout_ms: 3000 }, + }; + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.getByText("A classifier model is required")).toBeInTheDocument(); + }); + + it("does not show the classifier model error once a classifier model is set", () => { + const llmValue: ComplexityRouterConfigValue = { + ...defaultValue, + classifier_type: "llm", + classifier_llm_config: { model: "gpt-3.5-turbo", timeout_ms: 3000 }, + }; + renderWithProviders(); + fireEvent.click(screen.getByText("Advanced: Classification Method")); + expect(screen.queryByText("A classifier model is required")).not.toBeInTheDocument(); + }); + it("shows a validation error only under unfilled tiers when showValidationErrors is true", () => { renderWithProviders( = ({ label: model.model_group, })); + const classifierModelMissing = + showValidationErrors && value.classifier_type === "llm" && !value.classifier_llm_config?.model; + const handleTierChange = (tier: keyof ComplexityTiers, model: string) => { onChange({ ...value, @@ -233,7 +236,13 @@ const ComplexityRouterConfig: React.FC = ({ showSearch style={{ width: "100%" }} options={modelOptions} + status={classifierModelMissing ? "error" : undefined} /> + {classifierModelMissing && ( + + A classifier model is required + + )}