From d0e806d1b38bd5b553b9ca22c7b7ccda2d26a6a2 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 18 Nov 2025 19:49:16 -0800 Subject: [PATCH] Add additional model settings to chat models in test key (#16793) --- .../chat_ui/AdditionalModelSettings.test.tsx | 50 +++++++ .../chat_ui/AdditionalModelSettings.tsx | 137 ++++++++++++++++++ .../src/components/chat_ui/ChatUI.test.tsx | 8 +- .../src/components/chat_ui/ChatUI.tsx | 64 +++++++- .../llm_calls/chat_completion.test.tsx | 118 +++++++++++++++ .../chat_ui/llm_calls/chat_completion.tsx | 4 + 6 files changed, 375 insertions(+), 6 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/chat_ui/AdditionalModelSettings.test.tsx create mode 100644 ui/litellm-dashboard/src/components/chat_ui/AdditionalModelSettings.tsx create mode 100644 ui/litellm-dashboard/src/components/chat_ui/llm_calls/chat_completion.test.tsx diff --git a/ui/litellm-dashboard/src/components/chat_ui/AdditionalModelSettings.test.tsx b/ui/litellm-dashboard/src/components/chat_ui/AdditionalModelSettings.test.tsx new file mode 100644 index 00000000000..9eebc382163 --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat_ui/AdditionalModelSettings.test.tsx @@ -0,0 +1,50 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import AdditionalModelSettings from "./AdditionalModelSettings"; + +describe("AdditionalModelSettings", () => { + it("should render correctly", () => { + const { container } = render(); + expect(container).toBeTruthy(); + + expect(screen.getByText("Use Advanced Parameters")).toBeInTheDocument(); + + expect(screen.getByText("Temperature")).toBeInTheDocument(); + expect(screen.getByText("Max Tokens")).toBeInTheDocument(); + }); + + it("should enable the sliders when the checkbox is checked", async () => { + const user = userEvent.setup(); + const mockOnTemperatureChange = vi.fn(); + const mockOnMaxTokensChange = vi.fn(); + + render( + , + ); + + const checkbox = screen.getByRole("checkbox", { name: /Use Advanced Parameters/i }); + expect(checkbox).toBeInTheDocument(); + expect(checkbox).not.toBeChecked(); + + await user.click(checkbox); + + await waitFor(() => { + expect(screen.getByRole("checkbox", { name: /Use Advanced Parameters/i })).toBeChecked(); + }); + + await waitFor(() => { + const sliders = screen.getAllByRole("slider"); + expect(sliders.length).toBeGreaterThan(0); + expect(sliders[0]).not.toBeDisabled(); + }); + + const temperatureSlider = screen.getAllByRole("slider")[0]; + const maxTokensSlider = screen.getAllByRole("slider")[1]; + expect(temperatureSlider).not.toBeDisabled(); + expect(maxTokensSlider).not.toBeDisabled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/chat_ui/AdditionalModelSettings.tsx b/ui/litellm-dashboard/src/components/chat_ui/AdditionalModelSettings.tsx new file mode 100644 index 00000000000..85307509525 --- /dev/null +++ b/ui/litellm-dashboard/src/components/chat_ui/AdditionalModelSettings.tsx @@ -0,0 +1,137 @@ +import { InfoCircleOutlined } from "@ant-design/icons"; +import { Text } from "@tremor/react"; +import { Checkbox, InputNumber, Slider, Tooltip } from "antd"; +import React, { useEffect, useState } from "react"; + +interface AdditionalModelSettingsProps { + temperature?: number; + maxTokens?: number; + useAdvancedParams?: boolean; + onTemperatureChange?: (value: number) => void; + onMaxTokensChange?: (value: number) => void; + onUseAdvancedParamsChange?: (value: boolean) => void; +} + +const AdditionalModelSettings: React.FC = ({ + temperature = 1.0, + maxTokens = 2048, + useAdvancedParams: externalUseAdvancedParams, + onTemperatureChange, + onMaxTokensChange, + onUseAdvancedParamsChange, +}) => { + const [internalUseAdvancedParams, setInternalUseAdvancedParams] = useState(false); + const useAdvancedParams = + externalUseAdvancedParams !== undefined ? externalUseAdvancedParams : internalUseAdvancedParams; + const [localTemperature, setLocalTemperature] = useState(temperature); + const [localMaxTokens, setLocalMaxTokens] = useState(maxTokens); + + // Sync local state with props when they change + useEffect(() => { + setLocalTemperature(temperature); + }, [temperature]); + + useEffect(() => { + setLocalMaxTokens(maxTokens); + }, [maxTokens]); + + const handleTemperatureChange = (value: number | null) => { + const newValue = value ?? 1.0; + setLocalTemperature(newValue); + onTemperatureChange?.(newValue); + }; + + const handleMaxTokensChange = (value: number | null) => { + const newValue = value ?? 1000; + setLocalMaxTokens(newValue); + onMaxTokensChange?.(newValue); + }; + + const disabledOpacity = useAdvancedParams ? 1 : 0.4; + const disabledTextColor = useAdvancedParams ? "text-gray-700" : "text-gray-400"; + + const handleUseAdvancedParamsChange = (checked: boolean) => { + if (onUseAdvancedParamsChange) { + onUseAdvancedParamsChange(checked); + } else { + setInternalUseAdvancedParams(checked); + } + }; + + return ( +
+ handleUseAdvancedParamsChange(e.target.checked)}> + Use Advanced Parameters + + +
+
+
+
+ Temperature + + + +
+ +
+ +
+ +
+
+
+ Max Tokens + + + +
+ +
+ +
+
+
+ ); +}; + +export default AdditionalModelSettings; diff --git a/ui/litellm-dashboard/src/components/chat_ui/ChatUI.test.tsx b/ui/litellm-dashboard/src/components/chat_ui/ChatUI.test.tsx index 53fa2b0916e..9bf2d4fc14f 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/ChatUI.test.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/ChatUI.test.tsx @@ -120,7 +120,9 @@ describe("ChatUI", () => { // Open the "Select Model" dropdown (AntD renders options in a portal) const selectModelLabel = getByText("Select Model"); - const modelSelect = selectModelLabel.parentElement?.querySelector(".ant-select-selector"); + // The Select component is a sibling of the Text component, so we need to find it in the parent container + const modelSelectContainer = selectModelLabel.closest("div"); + const modelSelect = modelSelectContainer?.querySelector(".ant-select-selector"); expect(modelSelect).toBeTruthy(); fireEvent.mouseDown(modelSelect!); @@ -164,7 +166,9 @@ describe("ChatUI", () => { // Open model selector const selectModelLabel = getByText("Select Model"); - const modelSelect = selectModelLabel.parentElement?.querySelector(".ant-select-selector"); + // The Select component is a sibling of the Text component, so we need to find it in the parent container + const modelSelectContainer = selectModelLabel.closest("div"); + const modelSelect = modelSelectContainer?.querySelector(".ant-select-selector"); expect(modelSelect).toBeTruthy(); act(() => { fireEvent.mouseDown(modelSelect!); diff --git a/ui/litellm-dashboard/src/components/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/chat_ui/ChatUI.tsx index b65e254893f..211382541cc 100644 --- a/ui/litellm-dashboard/src/components/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui/ChatUI.tsx @@ -12,28 +12,30 @@ import { PictureOutlined, RobotOutlined, SafetyOutlined, + SettingOutlined, SoundOutlined, TagsOutlined, ToolOutlined, UserOutlined, } from "@ant-design/icons"; import { Card, Text, TextInput, Title, Button as TremorButton } from "@tremor/react"; -import { Button, Input, Modal, Select, Spin, Tooltip, Typography, Upload } from "antd"; +import { Button, Input, Modal, Popover, Select, Spin, Tooltip, Typography, Upload } from "antd"; import React, { useEffect, useRef, useState } from "react"; import ReactMarkdown from "react-markdown"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { coy } from "react-syntax-highlighter/dist/esm/styles/prism"; import { v4 as uuidv4 } from "uuid"; +import { truncateString } from "../../utils/textUtils"; import GuardrailSelector from "../guardrails/GuardrailSelector"; import NotificationsManager from "../molecules/notifications_manager"; import TagSelector from "../tag_management/TagSelector"; import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; +import AdditionalModelSettings from "./AdditionalModelSettings"; import AudioRenderer from "./AudioRenderer"; import { OPEN_AI_VOICE_SELECT_OPTIONS, OpenAIVoice } from "./chatConstants"; import ChatImageRenderer from "./ChatImageRenderer"; import ChatImageUpload from "./ChatImageUpload"; import { createChatDisplayMessage, createChatMultimodalMessage } from "./ChatImageUtils"; -import { truncateString } from "../../utils/textUtils"; import { generateCodeSnippet } from "./CodeSnippets"; import EndpointSelector from "./EndpointSelector"; import { makeAnthropicMessagesRequest } from "./llm_calls/anthropic_messages"; @@ -184,6 +186,9 @@ const ChatUI: React.FC = ({ const [generatedCode, setGeneratedCode] = useState(""); const [selectedSdk, setSelectedSdk] = useState<"openai" | "azure">("openai"); const [mcpEvents, setMCPEvents] = useState([]); + const [temperature, setTemperature] = useState(1.0); + const [maxTokens, setMaxTokens] = useState(2048); + const [useAdvancedParams, setUseAdvancedParams] = useState(false); const chatEndRef = useRef(null); @@ -765,6 +770,8 @@ const ChatUI: React.FC = ({ selectedMCPTools, // Pass the selected tools array updateChatImageUI, // Pass the image callback updateSearchResults, // Pass the search results callback + useAdvancedParams ? temperature : undefined, // Pass temperature if enabled + useAdvancedParams ? maxTokens : undefined, // Pass max_tokens if enabled ); } else if (endpointType === EndpointType.IMAGE) { // For image generation @@ -950,6 +957,19 @@ const ChatUI: React.FC = ({ setShowCustomModelInput(value === "custom"); }; + // Check if the selected model is a chat model + const isChatModel = () => { + if (!selectedModel || selectedModel === "custom") { + return false; + } + const model = modelInfo.find((m) => m.model_group === selectedModel); + if (!model) { + return false; + } + // Check if mode is explicitly "chat" or undefined (which defaults to chat per backend) + return !model.mode || model.mode === "chat"; + }; + const antIcon = ; return ( @@ -1037,8 +1057,44 @@ const ChatUI: React.FC = ({
- - Select Model + + + Select Model + + {isChatModel() ? ( + + } + title="Model Settings" + trigger="click" + placement="right" + > +