diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/AdditionalModelSettings.test.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/AdditionalModelSettings.test.tsx index 9eebc382163..9cd5770997b 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/AdditionalModelSettings.test.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/AdditionalModelSettings.test.tsx @@ -1,4 +1,4 @@ -import { render, screen, waitFor } from "@testing-library/react"; +import { act, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import AdditionalModelSettings from "./AdditionalModelSettings"; @@ -47,4 +47,55 @@ describe("AdditionalModelSettings", () => { expect(temperatureSlider).not.toBeDisabled(); expect(maxTokensSlider).not.toBeDisabled(); }); + + it("should not show Simulate failure to test fallbacks when onMockTestFallbacksChange is not provided", () => { + render(); + expect(screen.queryByText(/Simulate failure to test fallbacks/i)).not.toBeInTheDocument(); + }); + + it("should show and toggle Simulate failure to test fallbacks when callback is provided", async () => { + const user = userEvent.setup(); + const onMockTestFallbacksChange = vi.fn(); + let currentValue = false; + const handleChange = (value: boolean) => { + currentValue = value; + onMockTestFallbacksChange(value); + }; + + const { rerender } = render( + , + ); + + const fallbacksCheckbox = screen.getByRole("checkbox", { + name: /Simulate failure to test fallbacks/i, + }); + expect(fallbacksCheckbox).toBeInTheDocument(); + expect(fallbacksCheckbox).not.toBeChecked(); + + await act(async () => { + await user.click(fallbacksCheckbox); + }); + + await waitFor(() => { + expect(onMockTestFallbacksChange).toHaveBeenCalledWith(true); + }); + + rerender( + , + ); + + await act(async () => { + await user.click(screen.getByRole("checkbox", { name: /Simulate failure to test fallbacks/i })); + }); + + await waitFor(() => { + expect(onMockTestFallbacksChange).toHaveBeenCalledWith(false); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/AdditionalModelSettings.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/AdditionalModelSettings.tsx index 85307509525..a5fadb813b6 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/AdditionalModelSettings.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/AdditionalModelSettings.tsx @@ -10,6 +10,8 @@ interface AdditionalModelSettingsProps { onTemperatureChange?: (value: number) => void; onMaxTokensChange?: (value: number) => void; onUseAdvancedParamsChange?: (value: boolean) => void; + mockTestFallbacks?: boolean; + onMockTestFallbacksChange?: (value: boolean) => void; } const AdditionalModelSettings: React.FC = ({ @@ -19,6 +21,8 @@ const AdditionalModelSettings: React.FC = ({ onTemperatureChange, onMaxTokensChange, onUseAdvancedParamsChange, + mockTestFallbacks, + onMockTestFallbacksChange, }) => { const [internalUseAdvancedParams, setInternalUseAdvancedParams] = useState(false); const useAdvancedParams = @@ -64,6 +68,20 @@ const AdditionalModelSettings: React.FC = ({ Use Advanced Parameters + {onMockTestFallbacksChange && ( + +
+ onMockTestFallbacksChange(e.target.checked)} + > + Simulate failure to test fallbacks + + +
+
+ )} +
diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.test.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.test.tsx index 8d64d727189..e00f27c39f4 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.test.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.test.tsx @@ -271,6 +271,70 @@ describe("ChatUI", () => { }); }); + it("should show Simulate failure to test fallbacks in Model Settings when chat endpoint is selected", async () => { + render( + , + ); + + await waitFor(() => { + expect(screen.getByText("Test Key")).toBeInTheDocument(); + }); + + // Model Settings button only appears when a chat model is selected; select "Model 1" first + const selectModelLabel = screen.getByText("Select Model"); + const modelSelectContainer = selectModelLabel.closest("div"); + const modelSelect = modelSelectContainer?.querySelector(".ant-select-selector"); + expect(modelSelect).toBeTruthy(); + + await act(async () => { + fireEvent.mouseDown(modelSelect!); + }); + + await waitFor(() => { + expect(screen.getAllByText("Model 1").length).toBeGreaterThan(0); + }); + + // Ant Design Select options may not have role="option"; click the dropdown option by text + const model1Options = screen.getAllByText("Model 1"); + await act(async () => { + fireEvent.click(model1Options[model1Options.length - 1]); + }); + + await waitFor(() => { + const modelSettingsButton = screen.getByTestId("model-settings-button"); + expect(modelSettingsButton).toBeInTheDocument(); + }); + + const modelSettingsButton = screen.getByTestId("model-settings-button"); + await act(async () => { + fireEvent.click(modelSettingsButton); + }); + + await waitFor(() => { + expect(screen.getByText("Model Settings")).toBeInTheDocument(); + expect(screen.getByText(/Simulate failure to test fallbacks/i)).toBeInTheDocument(); + }); + + const fallbacksCheckbox = screen.getByRole("checkbox", { + name: /Simulate failure to test fallbacks/i, + }); + expect(fallbacksCheckbox).not.toBeChecked(); + + await act(async () => { + fireEvent.click(fallbacksCheckbox); + }); + + await waitFor(() => { + expect(screen.getByRole("checkbox", { name: /Simulate failure to test fallbacks/i })).toBeChecked(); + }); + }); + it("should show Fill button and populate customProxyBaseUrl when proxySettings.LITELLM_UI_API_DOC_BASE_URL is provided", async () => { const testProxyUrl = "http://localhost:5000"; diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index 92898c26bf2..7e50ca6b95a 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -229,6 +229,7 @@ const ChatUI: React.FC = ({ const [temperature, setTemperature] = useState(1.0); const [maxTokens, setMaxTokens] = useState(2048); const [useAdvancedParams, setUseAdvancedParams] = useState(false); + const [mockTestFallbacks, setMockTestFallbacks] = useState(false); // Code Interpreter state (using custom hook) const codeInterpreter = useCodeInterpreter(); @@ -982,6 +983,7 @@ const ChatUI: React.FC = ({ mcpServers, mcpServerToolRestrictions, handleMCPEvent, + mockTestFallbacks, ); } else if (endpointType === EndpointType.IMAGE) { // For image generation @@ -1401,6 +1403,8 @@ const ChatUI: React.FC = ({ onTemperatureChange={setTemperature} onMaxTokensChange={setMaxTokens} onUseAdvancedParamsChange={setUseAdvancedParams} + mockTestFallbacks={mockTestFallbacks} + onMockTestFallbacksChange={setMockTestFallbacks} /> } title="Model Settings" @@ -1412,6 +1416,8 @@ const ChatUI: React.FC = ({ size="small" icon={} className="text-gray-500 hover:text-gray-700" + aria-label="Model Settings" + data-testid="model-settings-button" /> ) : ( @@ -2390,7 +2396,7 @@ const ChatUI: React.FC = ({ setIsGetCodeModalVisible(false)} footer={null} width={800} diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.test.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.test.tsx index 8786e022013..8649834b318 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.test.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.test.tsx @@ -190,4 +190,70 @@ describe("chat_completion", () => { expect(secondTool.require_approval).toBe("never"); expect(secondTool.allowed_tools).toEqual(["toolC"]); }); + + it("should include mock_testing_fallbacks in request body when mockTestFallbacks is true", async () => { + await makeOpenAIChatCompletionRequest( + mockChatHistory, + mockUpdateUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + undefined, // onTimingData + undefined, // onUsageData + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // onImageGenerated + undefined, // onSearchResults + undefined, // temperature + undefined, // max_tokens + undefined, // onTotalLatency + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // onMCPEvent + true, // mockTestFallbacks + ); + + expect(mockCreate).toHaveBeenCalledTimes(1); + const callArgs = mockCreate.mock.calls[0][0]; + expect(callArgs.mock_testing_fallbacks).toBe(true); + }); + + it("should not include mock_testing_fallbacks in request body when mockTestFallbacks is false or undefined", async () => { + await makeOpenAIChatCompletionRequest( + mockChatHistory, + mockUpdateUI, + "gpt-4", + "test-token", + undefined, // tags + undefined, // signal + undefined, // onReasoningContent + undefined, // onTimingData + undefined, // onUsageData + undefined, // traceId + undefined, // vector_store_ids + undefined, // guardrails + undefined, // policies + undefined, // selectedMCPServers + undefined, // onImageGenerated + undefined, // onSearchResults + undefined, // temperature + undefined, // max_tokens + undefined, // onTotalLatency + undefined, // customBaseUrl + undefined, // mcpServers + undefined, // mcpServerToolRestrictions + undefined, // onMCPEvent + false, // mockTestFallbacks + ); + + expect(mockCreate).toHaveBeenCalledTimes(1); + const callArgs = mockCreate.mock.calls[0][0]; + expect(callArgs).not.toHaveProperty("mock_testing_fallbacks"); + }); }); diff --git a/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.tsx index 61d232082e0..048ea9bfa11 100644 --- a/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.tsx +++ b/ui/litellm-dashboard/src/components/playground/llm_calls/chat_completion.tsx @@ -30,6 +30,7 @@ export async function makeOpenAIChatCompletionRequest( mcpServers?: MCPServer[], mcpServerToolRestrictions?: Record, onMCPEvent?: (event: MCPEvent) => void, + mockTestFallbacks?: boolean, ) { // base url should be the current base_url const isLocal = process.env.NODE_ENV === "development"; @@ -115,6 +116,7 @@ export async function makeOpenAIChatCompletionRequest( ...(tools.length > 0 ? { tools, tool_choice: "auto" } : {}), ...(temperature !== undefined ? { temperature } : {}), ...(max_tokens !== undefined ? { max_tokens } : {}), + ...(mockTestFallbacks ? { mock_testing_fallbacks: true } : {}), }, { signal }, );