diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.test.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.test.tsx new file mode 100644 index 00000000000..0e05c141570 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.test.tsx @@ -0,0 +1,309 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import AddFallbacks, { Fallbacks } from "./AddFallbacks"; +import * as fetchModelsModule from "../../../playground/llm_calls/fetch_models"; + +vi.mock("../../../playground/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn(), +})); + +vi.mock("antd", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + message: { + error: vi.fn(), + }, + }; +}); + +vi.mock("./FallbackSelectionForm", () => ({ + FallbackSelectionForm: ({ groups, onGroupsChange }: any) => { + const handleUpdateGroup = () => { + if (groups.length > 0) { + const updatedGroups = groups.map((group: any, index: number) => { + if (index === 0 && !group.primaryModel) { + return { + ...group, + primaryModel: "gpt-4", + fallbackModels: ["gpt-3.5-turbo"], + }; + } + return group; + }); + onGroupsChange(updatedGroups); + } + }; + + return ( +
+ +
{groups.length}
+ {groups.map((group: any) => ( +
+ Primary: {group.primaryModel || "None"}, Fallbacks: {group.fallbackModels.length} +
+ ))} +
+ ); + }, +})); + +describe("AddFallbacks", () => { + const mockOnChange = vi.fn(); + const mockAccessToken = "test-token"; + const mockModelGroups = [ + { model_group: "gpt-4", mode: "chat" }, + { model_group: "gpt-3.5-turbo", mode: "chat" }, + { model_group: "claude-3-opus", mode: "chat" }, + ]; + + const defaultProps = { + accessToken: mockAccessToken, + value: [] as Fallbacks, + onChange: mockOnChange, + }; + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(fetchModelsModule.fetchAvailableModels).mockResolvedValue(mockModelGroups); + }); + + it("should render the component", () => { + render(); + expect(screen.getByRole("button", { name: /add fallbacks/i })).toBeInTheDocument(); + }); + + it("should open modal when Add Fallbacks button is clicked", async () => { + const user = userEvent.setup(); + render(); + + const addButton = screen.getByRole("button", { name: /add fallbacks/i }); + await user.click(addButton); + + await waitFor(() => { + expect(screen.getByRole("dialog")).toBeInTheDocument(); + }); + }); + + it("should fetch available models when modal opens", async () => { + const user = userEvent.setup(); + render(); + + const addButton = screen.getByRole("button", { name: /add fallbacks/i }); + await user.click(addButton); + + await waitFor(() => { + expect(fetchModelsModule.fetchAvailableModels).toHaveBeenCalledWith(mockAccessToken); + }); + }); + + it("should close modal when Cancel button is clicked", async () => { + const user = userEvent.setup(); + render(); + + const addButton = screen.getByRole("button", { name: /add fallbacks/i }); + await user.click(addButton); + + await waitFor(() => { + expect(screen.getByRole("dialog")).toBeInTheDocument(); + }); + + const cancelButton = screen.getByRole("button", { name: /cancel/i }); + await user.click(cancelButton); + + await waitFor(() => { + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + }); + + it("should show error when saving incomplete groups", async () => { + const user = userEvent.setup(); + const antd = await import("antd"); + render(); + + const addButton = screen.getByRole("button", { name: /add fallbacks/i }); + await user.click(addButton); + + await waitFor(() => { + expect(screen.getByRole("dialog")).toBeInTheDocument(); + }); + + await waitFor(() => { + const saveButton = screen.getByRole("button", { name: /save all configurations/i }); + expect(saveButton).toBeInTheDocument(); + }); + + const saveButton = screen.getByRole("button", { name: /save all configurations/i }); + await user.click(saveButton); + + await waitFor(() => { + expect(antd.message.error).toHaveBeenCalled(); + }); + }); + + it("should show error message when saving incomplete groups", async () => { + const user = userEvent.setup(); + const antd = await import("antd"); + render(); + + const addButton = screen.getByRole("button", { name: /add fallbacks/i }); + await user.click(addButton); + + await waitFor(() => { + expect(screen.getByRole("dialog")).toBeInTheDocument(); + }); + + const saveButton = screen.getByRole("button", { name: /save all configurations/i }); + await user.click(saveButton); + + await waitFor(() => { + expect(antd.message.error).toHaveBeenCalled(); + }); + }); + + it("should call onChange with new fallbacks when Save is clicked with valid configuration", async () => { + const user = userEvent.setup(); + mockOnChange.mockResolvedValue(undefined); + render(); + + const addButton = screen.getByRole("button", { name: /add fallbacks/i }); + await user.click(addButton); + + await waitFor(() => { + expect(screen.getByRole("dialog")).toBeInTheDocument(); + }); + + await waitFor(() => { + expect(screen.getByTestId("fallback-selection-form")).toBeInTheDocument(); + }); + + const updateGroupButton = screen.getByTestId("update-group-button"); + await user.click(updateGroupButton); + + await waitFor(() => { + expect(screen.getByText(/Primary: gpt-4/i)).toBeInTheDocument(); + }); + + const saveButton = screen.getByRole("button", { name: /save all configurations/i }); + expect(saveButton).not.toBeDisabled(); + await user.click(saveButton); + + await waitFor(() => { + expect(mockOnChange).toHaveBeenCalled(); + const callArgs = mockOnChange.mock.calls[0][0]; + expect(callArgs).toHaveLength(1); + expect(callArgs[0]).toHaveProperty("gpt-4"); + expect(callArgs[0]["gpt-4"]).toContain("gpt-3.5-turbo"); + }); + }); + + it("should append new fallbacks to existing value", async () => { + const user = userEvent.setup(); + const existingFallbacks: Fallbacks = [{ "existing-model": ["fallback-1"] }]; + mockOnChange.mockResolvedValue(undefined); + render(); + + const addButton = screen.getByRole("button", { name: /add fallbacks/i }); + await user.click(addButton); + + await waitFor(() => { + expect(screen.getByRole("dialog")).toBeInTheDocument(); + }); + + await waitFor(() => { + expect(screen.getByTestId("fallback-selection-form")).toBeInTheDocument(); + }); + + const updateGroupButton = screen.getByTestId("update-group-button"); + await user.click(updateGroupButton); + + await waitFor(() => { + expect(screen.getByText(/Primary: gpt-4/i)).toBeInTheDocument(); + }); + + const saveButton = screen.getByRole("button", { name: /save all configurations/i }); + await user.click(saveButton); + + await waitFor(() => { + expect(mockOnChange).toHaveBeenCalled(); + const callArgs = mockOnChange.mock.calls[0][0]; + expect(callArgs).toHaveLength(2); + expect(callArgs[0]).toEqual({ "existing-model": ["fallback-1"] }); + expect(callArgs[1]).toHaveProperty("gpt-4"); + }); + }); + + it("should reset form state when modal is closed", async () => { + const user = userEvent.setup(); + render(); + + const addButton = screen.getByRole("button", { name: /add fallbacks/i }); + await user.click(addButton); + + await waitFor(() => { + expect(screen.getByRole("dialog")).toBeInTheDocument(); + }); + + const cancelButton = screen.getByRole("button", { name: /cancel/i }); + await user.click(cancelButton); + + await waitFor(() => { + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + + await user.click(addButton); + + await waitFor(() => { + expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(screen.getByTestId("fallback-selection-form")).toBeInTheDocument(); + }); + }); + + it("should handle onChange error gracefully", async () => { + const user = userEvent.setup(); + const error = new Error("Save failed"); + mockOnChange.mockRejectedValue(error); + render(); + + const addButton = screen.getByRole("button", { name: /add fallbacks/i }); + await user.click(addButton); + + await waitFor(() => { + expect(screen.getByRole("dialog")).toBeInTheDocument(); + }); + + await waitFor(() => { + expect(screen.getByTestId("fallback-selection-form")).toBeInTheDocument(); + }); + + const updateGroupButton = screen.getByTestId("update-group-button"); + await user.click(updateGroupButton); + + await waitFor(() => { + expect(screen.getByText(/Primary: gpt-4/i)).toBeInTheDocument(); + }); + + const saveButton = screen.getByRole("button", { name: /save all configurations/i }); + await user.click(saveButton); + + await waitFor(() => { + expect(mockOnChange).toHaveBeenCalled(); + }); + }); + + it("should not call onChange when onChange prop is not provided", async () => { + const user = userEvent.setup(); + render(); + + const addButton = screen.getByRole("button", { name: /add fallbacks/i }); + await user.click(addButton); + + await waitFor(() => { + expect(screen.getByRole("dialog")).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx new file mode 100644 index 00000000000..c0b626cbb12 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacks.tsx @@ -0,0 +1,169 @@ +/** + * Parent component for adding fallbacks to the proxy router config + * Handles value/onChange logic and form submission + * Works with forms - reads from and writes to router_settings.fallbacks + */ + +import { Button as TremorButton } from "@tremor/react"; +import { Button, message } from "antd"; +import React, { useEffect, useState } from "react"; +import NotificationManager from "../../../molecules/notifications_manager"; +import { fetchAvailableModels, ModelGroup } from "../../../playground/llm_calls/fetch_models"; +import { AddFallbacksModal } from "./AddFallbacksModal"; +import { FallbackGroup } from "./FallbackGroupConfig"; +import { FallbackSelectionForm } from "./FallbackSelectionForm"; + +export type FallbackEntry = { [modelName: string]: string[] }; +export type Fallbacks = FallbackEntry[]; + +interface AddFallbacksProps { + models?: string[]; + accessToken: string; + value?: Fallbacks; // Current fallbacks value from form + onChange?: (fallbacks: Fallbacks) => Promise; // Callback to update form value +} + +export default function AddFallbacks({ + models, + accessToken, + value = [], + onChange, +}: AddFallbacksProps) { + const [isModalVisible, setIsModalVisible] = useState(false); + const [modelInfo, setModelInfo] = useState([]); + const [modalKey, setModalKey] = useState(0); // Key to force remount of form when modal opens + const [isSaving, setIsSaving] = useState(false); + const [groups, setGroups] = useState([ + { + id: "1", + primaryModel: null, + fallbackModels: [], + }, + ]); + + // Reset groups state and increment modal key when modal opens + useEffect(() => { + if (isModalVisible) { + setGroups([ + { + id: "1", + primaryModel: null, + fallbackModels: [], + }, + ]); + setModalKey((prev) => prev + 1); // Force remount of form + } + }, [isModalVisible]); + + useEffect(() => { + const loadModels = async () => { + try { + const uniqueModels = await fetchAvailableModels(accessToken); + console.log("Fetched models for fallbacks:", uniqueModels); + setModelInfo(uniqueModels); + } catch (error) { + console.error("Error fetching model info for fallbacks:", error); + } + }; + if (isModalVisible) { + loadModels(); + } + }, [accessToken, isModalVisible]); + + const availableModels = Array.from(new Set(modelInfo.map((option) => option.model_group))).sort(); + + const handleCancel = () => { + setIsModalVisible(false); + // Reset to initial state + setGroups([ + { + id: "1", + primaryModel: null, + fallbackModels: [], + }, + ]); + }; + + const handleSaveAll = async () => { + // Validation + const invalidGroups = groups.filter( + (g) => !g.primaryModel || g.fallbackModels.length === 0, + ); + if (invalidGroups.length > 0) { + message.error( + `Please complete configuration for all groups. ${invalidGroups.length} group(s) incomplete.`, + ); + return; + } + + // Create fallback objects in the format expected by the API + const newFallbacks = groups.map((g) => ({ + [g.primaryModel!]: g.fallbackModels, + })); + + // Get current fallbacks from form value, or an empty array if it's null/undefined + const currentFallbacks = value || []; + + // Add new fallbacks to the current fallbacks + const updatedFallbacks = [...currentFallbacks, ...newFallbacks]; + + // Call onChange to update the form value and wait for it to complete + if (onChange) { + setIsSaving(true); + try { + await onChange(updatedFallbacks); + NotificationManager.success(`${groups.length} fallback configuration(s) added successfully!`); + handleCancel(); + } catch (error) { + // Error handling is done in handleFallbacksChange, so we don't need to show another notification here + console.error("Error saving fallbacks:", error); + } finally { + setIsSaving(false); + } + } else { + NotificationManager.fromBackend("onChange callback not provided"); + } + }; + + return ( +
+ setIsModalVisible(true)} + icon={() => +} + > + Add Fallbacks + + + + {/* Footer with Cancel and Save buttons */} + {groups.length > 0 && ( +
+ + +
+ )} +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacksModal.test.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacksModal.test.tsx new file mode 100644 index 00000000000..c6be52f7c14 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacksModal.test.tsx @@ -0,0 +1,56 @@ +import { render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { AddFallbacksModal } from "./AddFallbacksModal"; + +describe("AddFallbacksModal", () => { + const mockOnCancel = vi.fn(); + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render the modal when open is true", () => { + render( + +
Test Content
+
, + ); + + expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(screen.getByText("Configure Model Fallbacks")).toBeInTheDocument(); + expect(screen.getByText("Manage multiple fallback chains for different models (up to 5 groups at a time)")).toBeInTheDocument(); + expect(screen.getByText("Test Content")).toBeInTheDocument(); + }); + + it("should not render the modal when open is false", () => { + render( + +
Test Content
+
, + ); + + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + + it("should render children content when modal is open", () => { + render( + +
Child Component
+
, + ); + + expect(screen.getByTestId("child-content")).toBeInTheDocument(); + expect(screen.getByText("Child Component")).toBeInTheDocument(); + }); + + it("should display the correct title and description", () => { + render( + +
Content
+
, + ); + + expect(screen.getByText("Configure Model Fallbacks")).toBeInTheDocument(); + expect(screen.getByText(/Manage multiple fallback chains/i)).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacksModal.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacksModal.tsx new file mode 100644 index 00000000000..5048c99d817 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/AddFallbacksModal.tsx @@ -0,0 +1,52 @@ +/** + * Modal wrapper for the fallback selection form + * Handles modal visibility and layout, but delegates content to children + */ + +import { Modal } from "antd"; +import { ArrowRight } from "lucide-react"; +import React from "react"; + +interface AddFallbacksModalProps { + open: boolean; + onCancel: () => void; + children: React.ReactNode; +} + +export function AddFallbacksModal({ + open, + onCancel, + children, +}: AddFallbacksModalProps) { + return ( + +
+
+ +
+
+

Configure Model Fallbacks

+

+ Manage multiple fallback chains for different models (up to 5 groups at a time) +

+
+
+ + } + open={open} + width={900} + footer={null} + onCancel={onCancel} + maskClosable={false} + className="top-8" + styles={{ + body: { padding: "24px" }, + header: { padding: "24px 24px 0 24px", border: "none" }, + }} + > +
{children}
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx new file mode 100644 index 00000000000..24ce80f0a0b --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackGroupConfig.tsx @@ -0,0 +1,208 @@ +/** + * Component for configuring a single fallback group + * Handles primary model selection and fallback chain configuration + */ + +import { Select, Tooltip } from "antd"; +import { AlertCircle, ArrowDown, X } from "lucide-react"; +import React from "react"; + +export interface FallbackGroup { + id: string; + primaryModel: string | null; + fallbackModels: string[]; +} + +interface FallbackGroupConfigProps { + group: FallbackGroup; + onChange: (updatedGroup: FallbackGroup) => void; + availableModels: string[]; + maxFallbacks: number; +} + +export function FallbackGroupConfig({ + group, + onChange, + availableModels, + maxFallbacks, +}: FallbackGroupConfigProps) { + // Filter available options for fallbacks (exclude primary only, allow already selected to be shown for deselection) + const availableFallbackOptions = availableModels.filter( + (m) => m !== group.primaryModel, + ); + + const handlePrimaryChange = (value: string) => { + let newFallbacks = [...group.fallbackModels]; + // Remove from fallbacks if it was there + if (newFallbacks.includes(value)) { + newFallbacks = newFallbacks.filter((m) => m !== value); + } + onChange({ + ...group, + primaryModel: value, + fallbackModels: newFallbacks, + }); + }; + + const handleFallbackSelect = (values: string[]) => { + // Limit to maxFallbacks + const limitedValues = values.slice(0, maxFallbacks); + + onChange({ + ...group, + fallbackModels: limitedValues, + }); + }; + + const removeFallback = (indexToRemove: number) => { + const newFallbacks = group.fallbackModels.filter((_, index) => index !== indexToRemove); + onChange({ + ...group, + fallbackModels: newFallbacks, + }); + }; + + const canAddMoreFallbacks = group.fallbackModels.length < maxFallbacks; + + return ( +
+ {/* Primary Model Section */} +
+ + ({ + label: m, + value: m, + }))} + optionRender={(option, info) => { + const isSelected = group.fallbackModels.includes(option.value as string); + const orderIndex = isSelected + ? group.fallbackModels.indexOf(option.value as string) + 1 + : null; + return ( +
+ {isSelected && orderIndex !== null && ( + + {orderIndex} + + )} + {option.label} +
+ ); + }} + maxTagCount="responsive" + maxTagPlaceholder={(omittedValues) => ( + value).join(", ")} + > + +{omittedValues.length} more + + )} + showSearch + filterOption={(input, option) => + (option?.label ?? "").toLowerCase().includes(input.toLowerCase()) + } + /> +

+ {canAddMoreFallbacks + ? `Search and select multiple models. Selected models will appear below in order. (${group.fallbackModels.length}/${maxFallbacks} used)` + : `Maximum ${maxFallbacks} fallbacks reached. Remove some to add more.`} +

+
+ + {/* Fallback List */} +
+ {group.fallbackModels.length === 0 ? ( +
+ No fallback models selected + Add models from the dropdown above +
+ ) : ( + group.fallbackModels.map((modelValue, index) => { + return ( +
+
+
+ {index + 1} +
+
+ {modelValue} +
+
+ + +
+ ); + }) + )} +
+
+ + + ); +} diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx new file mode 100644 index 00000000000..bb9ccb312a6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/FallbackSelectionForm.tsx @@ -0,0 +1,132 @@ +/** + * Form component for selecting and configuring fallback groups + * Manages groups state internally, but does not handle submission + * Decoupled from form submission logic + */ + +import { Button } from "@tremor/react"; +import { message, Tabs } from "antd"; +import { Plus } from "lucide-react"; +import React, { useEffect, useState } from "react"; +import { FallbackGroup, FallbackGroupConfig } from "./FallbackGroupConfig"; + +interface FallbackSelectionFormProps { + groups: FallbackGroup[]; + onGroupsChange: (groups: FallbackGroup[]) => void; + availableModels: string[]; + maxFallbacks?: number; + maxGroups?: number; +} + +export function FallbackSelectionForm({ + groups, + onGroupsChange, + availableModels, + maxFallbacks = 5, + maxGroups = 5, +}: FallbackSelectionFormProps) { + const [activeKey, setActiveKey] = useState(groups.length > 0 ? groups[0].id : "1"); + + // Reset activeKey when groups change (e.g., when modal reopens) + useEffect(() => { + if (groups.length > 0) { + // If current activeKey doesn't exist in groups, reset to first group + const activeKeyExists = groups.some((g) => g.id === activeKey); + if (!activeKeyExists) { + setActiveKey(groups[0].id); + } + } else { + // If groups is empty, reset activeKey + setActiveKey("1"); + } + }, [groups]); + + const handleAddGroup = () => { + if (groups.length >= maxGroups) { + return; + } + const newId = Date.now().toString(); + const newGroups = [ + ...groups, + { + id: newId, + primaryModel: null, + fallbackModels: [], + }, + ]; + onGroupsChange(newGroups); + setActiveKey(newId); + }; + + const handleRemoveGroup = (targetId: string) => { + if (groups.length === 1) { + message.warning("At least one group is required"); + return; + } + const newGroups = groups.filter((g) => g.id !== targetId); + onGroupsChange(newGroups); + if (activeKey === targetId && newGroups.length > 0) { + setActiveKey(newGroups[newGroups.length - 1].id); + } + }; + + const handleGroupUpdate = (updatedGroup: FallbackGroup) => { + const newGroups = groups.map((g) => (g.id === updatedGroup.id ? updatedGroup : g)); + onGroupsChange(newGroups); + }; + + // Generate tab items + const items = groups.map((group, index) => { + const label = group.primaryModel + ? group.primaryModel + : `Group ${index + 1}`; + return { + key: group.id, + label: label, + closable: groups.length > 1, // Only allow closing if there's more than 1 group + children: ( + + ), + }; + }); + + if (groups.length === 0) { + return ( +
+

No fallback groups configured

+ +
+ ); + } + + return ( + { + if (action === "add") handleAddGroup(); + else if (action === "remove" && groups.length > 1) { + handleRemoveGroup(targetKey as string); + } + }} + items={items} + className="fallback-tabs" + tabBarStyle={{ + marginBottom: 0, + }} + hideAdd={groups.length >= maxGroups} + /> + ); +} diff --git a/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx new file mode 100644 index 00000000000..bc6f6a4856c --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.test.tsx @@ -0,0 +1,372 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import Fallbacks from "./fallbacks"; +import * as networkingModule from "../../../networking"; +import * as fetchModelsModule from "../../../playground/llm_calls/fetch_models"; + +vi.mock("../../../networking", () => ({ + getCallbacksCall: vi.fn(), + setCallbacksCall: vi.fn(), +})); + +vi.mock("../../../playground/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn(), +})); + +vi.mock("openai", () => ({ + default: { + OpenAI: vi.fn().mockImplementation(() => ({ + chat: { + completions: { + create: vi.fn(), + }, + }, + })), + }, +})); + +vi.mock("../../../common_components/DeleteResourceModal", () => ({ + __esModule: true, + default: ({ isOpen, onOk, onCancel, title, message, resourceInformation, confirmLoading }: any) => { + if (!isOpen) return null; + return ( +
+
{title}
+
{message}
+ {resourceInformation?.map((info: any, idx: number) => ( +
+ {info.label}: {info.value} +
+ ))} + + +
+ ); + }, +})); + +vi.mock("./AddFallbacks", () => ({ + __esModule: true, + default: ({ value, onChange }: any) => { + const handleClick = async () => { + if (onChange) { + try { + const newFallbacks = [...(value || []), { "test-model": ["test-fallback"] }]; + await onChange(newFallbacks); + } catch (error) { + // Error is handled by the component + } + } + }; + return ( + + ); + }, +})); + +describe("Fallbacks", () => { + const mockAccessToken = "test-token"; + const mockUserRole = "Admin"; + const mockUserID = "user-123"; + const mockModelData = { + data: [ + { model_name: "gpt-4" }, + { model_name: "gpt-3.5-turbo" }, + { model_name: "claude-3-opus" }, + ], + }; + + const mockRouterSettings = { + fallbacks: [ + { "gpt-4": ["gpt-3.5-turbo", "claude-3-opus"] }, + { "claude-3-opus": ["gpt-4"] }, + ], + }; + + const defaultProps = { + accessToken: mockAccessToken, + userRole: mockUserRole, + userID: mockUserID, + modelData: mockModelData, + }; + + const findDeleteButton = (container: HTMLElement) => { + const tableRows = container.querySelectorAll("tbody tr"); + if (tableRows.length === 0) return null; + const firstRow = tableRows[0]; + const actionCells = firstRow.querySelectorAll("td"); + const lastCell = actionCells[actionCells.length - 1]; + const buttons = lastCell.querySelectorAll("button"); + if (buttons.length >= 2) { + return buttons[buttons.length - 1]; + } + const clickableElements = lastCell.querySelectorAll("[class*='cursor-pointer'], button"); + return Array.from(clickableElements).find((el) => + el.className.includes("red") || el.className.includes("hover:text-red") + ) || clickableElements[clickableElements.length - 1]; + }; + + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(networkingModule.getCallbacksCall).mockResolvedValue({ + router_settings: mockRouterSettings, + }); + vi.mocked(networkingModule.setCallbacksCall).mockResolvedValue(undefined); + vi.mocked(fetchModelsModule.fetchAvailableModels).mockResolvedValue([ + { model_group: "gpt-4", mode: "chat" }, + { model_group: "gpt-3.5-turbo", mode: "chat" }, + { model_group: "claude-3-opus", mode: "chat" }, + ]); + }); + + it("should render the component", async () => { + render(); + + await waitFor(() => { + expect(screen.getByTestId("add-fallbacks-button")).toBeInTheDocument(); + }); + }); + + it("should not render when accessToken is null", () => { + const { container } = render(); + expect(container.firstChild).toBeNull(); + }); + + it("should fetch router settings on mount", async () => { + render(); + + await waitFor(() => { + expect(networkingModule.getCallbacksCall).toHaveBeenCalledWith( + mockAccessToken, + mockUserID, + mockUserRole, + ); + }); + }); + + it("should display fallback entries in table", async () => { + render(); + + await waitFor(() => { + expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); + expect(screen.getByText("gpt-3.5-turbo, claude-3-opus")).toBeInTheDocument(); + expect(screen.getByText("claude-3-opus")).toBeInTheDocument(); + }); + }); + + it("should open delete modal when delete icon is clicked", async () => { + const user = userEvent.setup(); + const { container } = render(); + + await waitFor(() => { + expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); + }); + + const deleteButton = findDeleteButton(container); + expect(deleteButton).not.toBeNull(); + + await user.click(deleteButton as HTMLElement); + + await waitFor(() => { + expect(screen.getByTestId("delete-modal")).toBeInTheDocument(); + expect(screen.getByText("Delete Fallback?")).toBeInTheDocument(); + }); + }); + + it("should delete fallback when confirmed", async () => { + const user = userEvent.setup(); + const { container } = render(); + + await waitFor(() => { + expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); + }); + + const deleteButton = findDeleteButton(container); + expect(deleteButton).not.toBeNull(); + + await user.click(deleteButton as HTMLElement); + + await waitFor(() => { + expect(screen.getByTestId("delete-modal")).toBeInTheDocument(); + }); + + const confirmButton = screen.getByRole("button", { name: /delete/i }); + await user.click(confirmButton); + + await waitFor(() => { + expect(networkingModule.setCallbacksCall).toHaveBeenCalled(); + const callArgs = networkingModule.setCallbacksCall.mock.calls[0]; + expect(callArgs[0]).toBe(mockAccessToken); + expect(callArgs[1].router_settings.fallbacks).toHaveLength(1); + }); + }); + + it("should close delete modal when cancel is clicked", async () => { + const user = userEvent.setup(); + const { container } = render(); + + await waitFor(() => { + expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); + }); + + const deleteButton = findDeleteButton(container); + expect(deleteButton).not.toBeNull(); + + await user.click(deleteButton as HTMLElement); + + await waitFor(() => { + expect(screen.getByTestId("delete-modal")).toBeInTheDocument(); + }); + + const cancelButton = screen.getByRole("button", { name: /cancel/i }); + await user.click(cancelButton); + + await waitFor(() => { + expect(screen.queryByTestId("delete-modal")).not.toBeInTheDocument(); + }); + }); + + it("should show error notification on delete failure", async () => { + const user = userEvent.setup(); + const error = new Error("Delete failed"); + vi.mocked(networkingModule.setCallbacksCall).mockRejectedValueOnce(error); + const { container } = render(); + + await waitFor(() => { + expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); + }); + + const deleteButton = findDeleteButton(container); + expect(deleteButton).not.toBeNull(); + + await user.click(deleteButton as HTMLElement); + + await waitFor(() => { + expect(screen.getByTestId("delete-modal")).toBeInTheDocument(); + }); + + const confirmButton = screen.getByRole("button", { name: /delete/i }); + await user.click(confirmButton); + + await waitFor(() => { + expect(networkingModule.setCallbacksCall).toHaveBeenCalled(); + }); + }); + + it("should handle delete error gracefully", async () => { + const user = userEvent.setup(); + const error = new Error("Delete failed"); + vi.mocked(networkingModule.setCallbacksCall).mockRejectedValueOnce(error); + const { container } = render(); + + await waitFor(() => { + expect(screen.getAllByText("gpt-4").length).toBeGreaterThan(0); + }); + + const deleteButton = findDeleteButton(container); + expect(deleteButton).not.toBeNull(); + + await user.click(deleteButton as HTMLElement); + + await waitFor(() => { + expect(screen.getByTestId("delete-modal")).toBeInTheDocument(); + }); + + const confirmButton = screen.getByRole("button", { name: /delete/i }); + await user.click(confirmButton); + + await waitFor(() => { + expect(networkingModule.setCallbacksCall).toHaveBeenCalled(); + expect(screen.queryByTestId("delete-modal")).not.toBeInTheDocument(); + }); + }); + + it("should handle empty fallbacks array", async () => { + vi.mocked(networkingModule.getCallbacksCall).mockResolvedValueOnce({ + router_settings: { fallbacks: [] }, + }); + render(); + + await waitFor(() => { + expect(screen.getByTestId("add-fallbacks-button")).toBeInTheDocument(); + }); + + expect(screen.queryByText("gpt-4")).not.toBeInTheDocument(); + }); + + it("should handle router settings without fallbacks property", async () => { + vi.mocked(networkingModule.getCallbacksCall).mockResolvedValueOnce({ + router_settings: {}, + }); + render(); + + await waitFor(() => { + expect(screen.getByTestId("add-fallbacks-button")).toBeInTheDocument(); + }); + }); + + it("should remove model_group_retry_policy from router settings", async () => { + vi.mocked(networkingModule.getCallbacksCall).mockResolvedValueOnce({ + router_settings: { + ...mockRouterSettings, + model_group_retry_policy: { some: "policy" }, + }, + }); + render(); + + await waitFor(() => { + expect(networkingModule.getCallbacksCall).toHaveBeenCalled(); + }); + }); + + it("should update fallbacks when AddFallbacks onChange is called", async () => { + const user = userEvent.setup(); + render(); + + await waitFor(() => { + expect(screen.getByTestId("add-fallbacks-button")).toBeInTheDocument(); + }); + + const addButton = screen.getByTestId("add-fallbacks-button"); + await user.click(addButton); + + await waitFor(() => { + expect(networkingModule.setCallbacksCall).toHaveBeenCalled(); + }); + }); + + it("should handle fallbacks change error and refetch", async () => { + const user = userEvent.setup(); + const error = new Error("Update failed"); + vi.mocked(networkingModule.setCallbacksCall).mockRejectedValueOnce(error); + vi.mocked(networkingModule.getCallbacksCall).mockResolvedValue({ + router_settings: mockRouterSettings, + }); + render(); + + await waitFor(() => { + expect(screen.getByTestId("add-fallbacks-button")).toBeInTheDocument(); + }); + + const addButton = screen.getByTestId("add-fallbacks-button"); + await user.click(addButton); + + await waitFor(() => { + expect(networkingModule.setCallbacksCall).toHaveBeenCalled(); + }); + + await waitFor( + () => { + expect(networkingModule.getCallbacksCall).toHaveBeenCalledTimes(2); + }, + { timeout: 3000 }, + ); + }); +}); diff --git a/ui/litellm-dashboard/src/components/fallbacks.tsx b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx similarity index 81% rename from ui/litellm-dashboard/src/components/fallbacks.tsx rename to ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx index dc90ae79136..f493cc51323 100644 --- a/ui/litellm-dashboard/src/components/fallbacks.tsx +++ b/ui/litellm-dashboard/src/components/Settings/RouterSettings/Fallbacks/Fallbacks.tsx @@ -3,10 +3,10 @@ import { Icon, Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow import { Tooltip } from "antd"; import openai from "openai"; import React, { useEffect, useState } from "react"; -import AddFallbacks from "./add_fallbacks"; -import DeleteResourceModal from "./common_components/DeleteResourceModal"; -import NotificationsManager from "./molecules/notifications_manager"; -import { getCallbacksCall, setCallbacksCall } from "./networking"; +import DeleteResourceModal from "../../../common_components/DeleteResourceModal"; +import NotificationsManager from "../../../molecules/notifications_manager"; +import { getCallbacksCall, setCallbacksCall } from "../../../networking"; +import AddFallbacks from "./AddFallbacks"; type FallbackEntry = { [modelName: string]: string[] }; type Fallbacks = FallbackEntry[]; @@ -21,7 +21,7 @@ interface FallbacksProps { async function testFallbackModelResponse(selectedModel: string, accessToken: string) { const isLocal = process.env.NODE_ENV === "development"; if (isLocal != true) { - console.log = function () {}; + console.log = function () { }; } const proxyBaseUrl = isLocal ? "http://localhost:4000" : window.location.origin; const client = new openai.OpenAI({ @@ -142,13 +142,48 @@ const Fallbacks: React.FC = ({ accessToken, userRole, userID, mo return null; } + const handleFallbacksChange = async (fallbacks: Fallbacks): Promise => { + if (!accessToken) { + return; + } + + const updatedSettings = { + ...routerSettings, + fallbacks: fallbacks, + }; + + const payload = { + router_settings: updatedSettings, + }; + + try { + await setCallbacksCall(accessToken, payload); + // Update UI only after successful API call + setRouterSettings(updatedSettings); + } catch (error) { + // Revert on error by refetching from server + NotificationsManager.fromBackend("Failed to update router settings: " + error); + if (accessToken && userRole && userID) { + getCallbacksCall(accessToken, userID, userRole).then((data) => { + let router_settings = data.router_settings; + if ("model_group_retry_policy" in router_settings) { + delete router_settings["model_group_retry_policy"]; + } + setRouterSettings(router_settings); + }); + } + // Re-throw error so caller can handle it + throw error; + } + }; + return ( <> data.model_name) : []} - accessToken={accessToken} - routerSettings={routerSettings} - setRouterSettings={setRouterSettings} + accessToken={accessToken || ""} + value={routerSettings.fallbacks || []} + onChange={handleFallbacksChange} /> diff --git a/ui/litellm-dashboard/src/components/add_fallbacks.test.tsx b/ui/litellm-dashboard/src/components/add_fallbacks.test.tsx deleted file mode 100644 index 102b55af8af..00000000000 --- a/ui/litellm-dashboard/src/components/add_fallbacks.test.tsx +++ /dev/null @@ -1,47 +0,0 @@ -import { render, screen } from "@testing-library/react"; -import { beforeEach, describe, expect, it, vi } from "vitest"; -import AddFallbacks from "./add_fallbacks"; - -vi.mock("./networking", () => ({ - setCallbacksCall: vi.fn(), -})); - -vi.mock("./playground/llm_calls/fetch_models", () => ({ - fetchAvailableModels: vi.fn(() => - Promise.resolve([ - { model_group: "gpt-4" }, - { model_group: "gpt-3.5-turbo" }, - { model_group: "claude-3-opus" }, - { model_group: "claude-3-sonnet" }, - ]), - ), -})); - -vi.mock("./molecules/notifications_manager", () => ({ - default: { - success: vi.fn(), - fromBackend: vi.fn(), - }, -})); - -describe("AddFallbacks", () => { - const mockAccessToken = "test-token"; - const mockRouterSettings = { fallbacks: [] }; - const mockSetRouterSettings = vi.fn(); - - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("should render the component", () => { - render( - , - ); - - expect(screen.getByRole("button", { name: /Add Fallbacks/i })).toBeInTheDocument(); - }); -}); diff --git a/ui/litellm-dashboard/src/components/add_fallbacks.tsx b/ui/litellm-dashboard/src/components/add_fallbacks.tsx deleted file mode 100644 index 97b020005ae..00000000000 --- a/ui/litellm-dashboard/src/components/add_fallbacks.tsx +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Modal to add fallbacks to the proxy router config - */ - -import { Button } from "@tremor/react"; -import { Form, Modal, Select } from "antd"; -import React, { useEffect, useState } from "react"; -import NotificationManager from "./molecules/notifications_manager"; -import { setCallbacksCall } from "./networking"; -import { fetchAvailableModels, ModelGroup } from "./playground/llm_calls/fetch_models"; - -interface AddFallbacksProps { - models?: string[]; - accessToken: string; - routerSettings: { [key: string]: any }; - setRouterSettings: React.Dispatch>; -} - -const AddFallbacks: React.FC = ({ models, accessToken, routerSettings, setRouterSettings }) => { - const [form] = Form.useForm(); - const [isModalVisible, setIsModalVisible] = useState(false); - const [selectedModel, setSelectedModel] = useState(""); - const [modelInfo, setModelInfo] = useState([]); - const [selectedFallbacks, setSelectedFallbacks] = useState([]); - - useEffect(() => { - const loadModels = async () => { - try { - const uniqueModels = await fetchAvailableModels(accessToken); - console.log("Fetched models for fallbacks:", uniqueModels); - setModelInfo(uniqueModels); - } catch (error) { - console.error("Error fetching model info for fallbacks:", error); - } - }; - loadModels(); - }, [accessToken]); - const handleOk = () => { - setIsModalVisible(false); - form.resetFields(); - setSelectedFallbacks([]); - setSelectedModel(""); - }; - - const handleCancel = () => { - setIsModalVisible(false); - form.resetFields(); - setSelectedFallbacks([]); - setSelectedModel(""); - }; - - const updateFallbacks = (formValues: Record) => { - // Print the received value - console.log(formValues); - - // Extract model_name and models from formValues - const { model_name, models } = formValues; - - // Create new fallback - const newFallback = { [model_name]: models }; - - // Get current fallbacks, or an empty array if it's null - const currentFallbacks = routerSettings.fallbacks || []; - - // Add new fallback to the current fallbacks - const updatedFallbacks = [...currentFallbacks, newFallback]; - - // Create a new routerSettings object with updated fallbacks - const updatedRouterSettings = { ...routerSettings, fallbacks: updatedFallbacks }; - - // Print updated routerSettings - console.log(updatedRouterSettings); - - const payload = { - router_settings: updatedRouterSettings, - }; - - try { - setCallbacksCall(accessToken, payload); - // Update routerSettings state - setRouterSettings(updatedRouterSettings); - } catch (error) { - NotificationManager.fromBackend("Failed to update router settings: " + error); - } - - NotificationManager.success("router settings updated successfully"); - - setIsModalVisible(false); - form.resetFields(); - setSelectedFallbacks([]); - setSelectedModel(""); - }; - - return ( -
- - -

Add Fallbacks

-
- } - open={isModalVisible} - width={900} - footer={null} - onOk={handleOk} - onCancel={handleCancel} - className="top-8" - styles={{ - body: { padding: "24px" }, - header: { padding: "24px 24px 0 24px", border: "none" }, - }} - > -
-
-

- Configure fallback models to improve reliability. When the primary model fails or is unavailable, requests - will automatically route to the specified fallback models in order. -

-
- -
-
- - Primary Model * - - } - name="model_name" - rules={[{ required: true, message: "Please select the primary model that needs fallbacks" }]} - className="!mb-0" - > - -

This is the primary model that users will request

-
- -
- - - Fallback Models (select multiple) * - - } - name="models" - rules={[{ required: true, message: "Please select at least one fallback model" }]} - className="!mb-0" - > -
- {/* Show selected models in order */} - {selectedFallbacks.length > 0 && ( -
-

Fallback Order:

-
- {selectedFallbacks.map((model, index) => ( -
- {index + 1}. - {model} - -
- ))} -
-
- )} - - {/* Model selector */} - -
-

- Order matters: Models will be tried in the order shown above (1st, 2nd, 3rd, etc.) -

-
-
- -
- - -
- -
- - - ); -}; - -export default AddFallbacks; diff --git a/ui/litellm-dashboard/src/components/fallbacks.test.tsx b/ui/litellm-dashboard/src/components/fallbacks.test.tsx deleted file mode 100644 index e6db11270cd..00000000000 --- a/ui/litellm-dashboard/src/components/fallbacks.test.tsx +++ /dev/null @@ -1,95 +0,0 @@ -import { render, screen, waitFor } from "@testing-library/react"; -import { beforeEach, describe, expect, it, vi } from "vitest"; -import Fallbacks from "./fallbacks"; -import { getCallbacksCall, setCallbacksCall } from "./networking"; - -vi.mock("./networking", () => ({ - getCallbacksCall: vi.fn(), - setCallbacksCall: vi.fn(), -})); - -vi.mock("./add_fallbacks", () => ({ - __esModule: true, - default: () =>
Mock Add Fallbacks
, -})); - -vi.mock("openai", () => ({ - default: { - OpenAI: vi.fn().mockImplementation(() => ({ - chat: { - completions: { - create: vi.fn().mockResolvedValue({ - model: "test-model", - }), - }, - }, - })), - }, -})); - -describe("Fallbacks", () => { - const defaultProps = { - accessToken: "token", - userRole: "admin", - userID: "user-123", - modelData: { data: [] }, - }; - const mockGetCallbacksCall = vi.mocked(getCallbacksCall); - const mockSetCallbacksCall = vi.mocked(setCallbacksCall); - - beforeEach(() => { - vi.clearAllMocks(); - mockGetCallbacksCall.mockResolvedValue({ - router_settings: { - fallbacks: [], - }, - }); - mockSetCallbacksCall.mockResolvedValue({}); - }); - - it("should render", async () => { - render(); - - await waitFor(() => { - expect(screen.getByRole("columnheader", { name: "Model Name" })).toBeInTheDocument(); - }); - }); - - it("should render fallback data when callback data is returned from network call", async () => { - const mockFallbackData = { - router_settings: { - fallbacks: [{ "xai/grok-2": ["xai/grok-4", "gpt-4"] }, { "gpt-3.5-turbo": ["gpt-4"] }], - }, - }; - - mockGetCallbacksCall.mockResolvedValue(mockFallbackData); - - render(); - - await waitFor(() => { - expect(screen.getByText("xai/grok-2")).toBeInTheDocument(); - expect(screen.getByText("xai/grok-4, gpt-4")).toBeInTheDocument(); - expect(screen.getByText("gpt-3.5-turbo")).toBeInTheDocument(); - expect(screen.getByText("gpt-4")).toBeInTheDocument(); - }); - - expect(mockGetCallbacksCall).toHaveBeenCalledWith( - defaultProps.accessToken, - defaultProps.userID, - defaultProps.userRole, - ); - }); - - it("should render AddFallbacks component", async () => { - render(); - - await waitFor(() => { - expect(screen.getByText("Mock Add Fallbacks")).toBeInTheDocument(); - }); - }); - - it("should not render when access token is not provided", () => { - const { container } = render(); - expect(container.firstChild).toBeNull(); - }); -}); diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index 1dc35550558..18f891705e1 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -22,7 +22,7 @@ import { InputNumber } from "antd"; import { TrashIcon, CheckCircleIcon } from "@heroicons/react/outline"; import RouterSettings from "./router_settings"; -import Fallbacks from "./fallbacks"; +import Fallbacks from "./Settings/RouterSettings/Fallbacks/Fallbacks"; interface GeneralSettingsPageProps { accessToken: string | null; userRole: string | null;