From 74673fa959f46f93d3b21fd99a1f1cb5edc166df Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 21 Feb 2026 13:11:53 -0800 Subject: [PATCH] feat(agentbuilderview.tsx): run compliance datasets against litellm agent --- .../src/app/(dashboard)/playground/page.tsx | 5 +- .../playground/chat_ui/AgentBuilderView.tsx | 177 ++++-------- .../components/playground/chat_ui/ChatUI.tsx | 71 +++-- .../playground/complianceUI/ComplianceUI.tsx | 266 +++++++++++++----- 4 files changed, 297 insertions(+), 222 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx index 88570130a6e..555930a576c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/page.tsx @@ -40,7 +40,7 @@ export default function PlaygroundPage() { Chat Compare Compliance - Agent Builder + Agent Builder (Experimental) @@ -62,8 +62,11 @@ export default function PlaygroundPage() { diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx index f8e3c00cdc0..10719b7ff4f 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx @@ -2,21 +2,26 @@ import { CommentOutlined, ExperimentOutlined, PlusOutlined, RobotOutlined, SaveOutlined } from "@ant-design/icons"; import { Button, Input, Select, Spin, Tabs } from "antd"; -import React, { useCallback, useEffect, useRef, useState } from "react"; +import React, { useCallback, useEffect, useState } from "react"; import NotificationsManager from "../../molecules/notifications_manager"; import { modelCreateCall } from "../../networking"; -import { makeOpenAIChatCompletionRequest } from "../llm_calls/chat_completion"; import { AgentModel, fetchAvailableAgentModels } from "../llm_calls/fetch_agents"; import { fetchAvailableModels, ModelGroup } from "../llm_calls/fetch_models"; -import { createDisplayMessage } from "./ResponsesImageUtils"; -import { MessageType } from "./types"; +import ComplianceUI from "../complianceUI/ComplianceUI"; +import ChatUI from "./ChatUI"; const { TextArea } = Input; export interface AgentBuilderViewProps { accessToken: string | null; + token: string | null; userID: string | null; userRole: string | null; + disabledPersonalKeyCreation?: boolean; + proxySettings?: { + PROXY_BASE_URL?: string; + LITELLM_UI_API_DOC_BASE_URL?: string | null; + }; apiKey?: string; customProxyBaseUrl?: string; } @@ -25,8 +30,11 @@ const NEW_AGENT_ID = "__new__"; export default function AgentBuilderView({ accessToken, + token, userID, userRole, + disabledPersonalKeyCreation = false, + proxySettings, apiKey, customProxyBaseUrl, }: AgentBuilderViewProps) { @@ -43,12 +51,7 @@ export default function AgentBuilderView({ const [draftTemperature, setDraftTemperature] = useState(0.7); const [draftMaxTokens, setDraftMaxTokens] = useState(4096); - // Chat state (for Chat tab) - const [chatHistory, setChatHistory] = useState([]); - const [chatInput, setChatInput] = useState(""); - const [chatLoading, setChatLoading] = useState(false); const [saving, setSaving] = useState(false); - const abortControllerRef = useRef(null); const effectiveApiKey = apiKey || accessToken || ""; const selectedAgent = selectedId === NEW_AGENT_ID ? null : agentModels.find((a) => a.model_name === selectedId) ?? null; @@ -130,65 +133,6 @@ export default function AgentBuilderView({ } }; - const updateTextUI = useCallback((role: string, chunk: string, model?: string) => { - setChatHistory((prev) => { - const last = prev[prev.length - 1]; - if (last && last.role === role && !last.isImage && !last.isAudio) { - return [ - ...prev.slice(0, -1), - { ...last, content: (last.content as string) + chunk, model: last.model ?? model }, - ]; - } - return [...prev, { role, content: chunk, model } as MessageType]; - }); - }, []); - - const handleSendMessage = async () => { - const text = chatInput.trim(); - if (!text || !selectedAgent || !effectiveApiKey) return; - const displayMessage = createDisplayMessage(text, false); - setChatHistory((prev) => [...prev, displayMessage]); - setChatInput(""); - setChatLoading(true); - abortControllerRef.current = new AbortController(); - const apiHistory = [ - ...chatHistory - .filter((m) => !m.isImage && !m.isAudio) - .map((m) => ({ role: m.role, content: typeof m.content === "string" ? m.content : "" })), - { role: "user" as const, content: text }, - ]; - try { - await makeOpenAIChatCompletionRequest( - apiHistory, - (chunk, model) => updateTextUI("assistant", chunk, model), - selectedAgent.model_name, - effectiveApiKey, - undefined, - abortControllerRef.current.signal, - undefined, - undefined, - undefined, - undefined, - undefined, - undefined, - undefined, - undefined, - undefined, - undefined, - undefined, - 0.7, - 4096, - undefined, - customProxyBaseUrl, - ); - } catch (e) { - NotificationsManager.fromBackend("Chat request failed"); - updateTextUI("assistant", "Error: request failed."); - } finally { - setChatLoading(false); - } - }; - if (!accessToken || !userID || !userRole) { return (
@@ -199,8 +143,9 @@ export default function AgentBuilderView({ return (
-
- Agent Builder +
+
+ Agent Builder {isNewAgent ? (
+
+ + + Agent Builder is experimental and may change or be removed without notice. + +
@@ -234,10 +186,7 @@ export default function AgentBuilderView({ -
-
- + ) : (
Save an agent first to test in Chat. @@ -435,9 +349,22 @@ export default function AgentBuilderView({ Batch Test ), + disabled: isNewAgent, children: ( -
- Batch Test placeholder. Select an agent and use Chat to test. +
+ {selectedAgent ? ( + + ) : ( +
+ Select an agent to run batch tests. +
+ )}
), }, diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx index 7e50ca6b95a..4a47cc00b20 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/ChatUI.tsx @@ -84,6 +84,10 @@ interface ChatUIProps { PROXY_BASE_URL?: string; LITELLM_UI_API_DOC_BASE_URL?: string | null; }; + /** When true, hide configuration sidebar and use fixedModel only (e.g. embedded in Agent Builder). */ + simplified?: boolean; + /** When simplified is true, use this as the model and do not show model selector. */ + fixedModel?: string; } const MCP_SUPPORTED_ENDPOINTS = new Set([ @@ -99,6 +103,8 @@ const ChatUI: React.FC = ({ userID, disabledPersonalKeyCreation, proxySettings, + simplified = false, + fixedModel, }) => { const [mcpServers, setMCPServers] = useState([]); const [selectedMCPServers, setSelectedMCPServers] = useState(() => { @@ -140,6 +146,7 @@ const ChatUI: React.FC = ({ ); const [inputMessage, setInputMessage] = useState(""); const [chatHistory, setChatHistory] = useState(() => { + if (simplified) return []; try { const saved = sessionStorage.getItem("chatHistory"); return saved ? JSON.parse(saved) : []; @@ -148,7 +155,7 @@ const ChatUI: React.FC = ({ return []; } }); - const [selectedModel, setSelectedModel] = useState(undefined); + const [selectedModel, setSelectedModel] = useState(simplified ? fixedModel : undefined); const [showCustomModelInput, setShowCustomModelInput] = useState(false); const [modelInfo, setModelInfo] = useState([]); const [agentInfo, setAgentInfo] = useState([]); @@ -252,6 +259,14 @@ const ChatUI: React.FC = ({ } }; + // When simplified, keep selectedModel and endpointType in sync with fixedModel / chat-only + useEffect(() => { + if (simplified && fixedModel) { + setSelectedModel(fixedModel); + setEndpointType(EndpointType.CHAT); + } + }, [simplified, fixedModel]); + // Fetch tools for a specific server const loadServerTools = async (serverId: string) => { const userApiKey = apiKeySource === "session" ? accessToken : apiKey; @@ -312,6 +327,7 @@ const ChatUI: React.FC = ({ ]); useEffect(() => { + if (simplified) return; // Do not persist chat history in simplified (embedded) mode const handler = setTimeout(() => { sessionStorage.setItem("chatHistory", JSON.stringify(chatHistory)); }, 500); // Debounce by 500ms @@ -319,7 +335,7 @@ const ChatUI: React.FC = ({ return () => { clearTimeout(handler); }; - }, [chatHistory]); + }, [chatHistory, simplified]); useEffect(() => { sessionStorage.setItem("apiKeySource", JSON.stringify(apiKeySource)); @@ -334,10 +350,12 @@ const ChatUI: React.FC = ({ sessionStorage.setItem("selectedVoice", selectedVoice); sessionStorage.removeItem("selectedMCPTools"); // Clean up old key - if (selectedModel) { - sessionStorage.setItem("selectedModel", selectedModel); - } else { - sessionStorage.removeItem("selectedModel"); + if (!simplified) { + if (selectedModel) { + sessionStorage.setItem("selectedModel", selectedModel); + } else { + sessionStorage.removeItem("selectedModel"); + } } if (messageTraceId) { sessionStorage.setItem("messageTraceId", messageTraceId); @@ -352,6 +370,7 @@ const ChatUI: React.FC = ({ sessionStorage.setItem("useApiSessionManagement", JSON.stringify(useApiSessionManagement)); // Note: codeInterpreterEnabled and selectedContainerId are persisted by useCodeInterpreter hook }, [ + simplified, apiKeySource, apiKey, selectedModel, @@ -375,7 +394,7 @@ const ChatUI: React.FC = ({ return; } - // Fetch model info and set the default selected model + // Fetch model info and set the default selected model (skip in simplified mode; we use fixedModel) const loadModels = async () => { try { if (!userApiKey) { @@ -400,9 +419,11 @@ const ChatUI: React.FC = ({ } }; - loadModels(); + if (!simplified) { + loadModels(); + } loadMCPServers(); - }, [accessToken, userID, userRole, apiKeySource, apiKey, token]); + }, [accessToken, userID, userRole, apiKeySource, apiKey, token, simplified]); // Load tools when MCP direct mode has a server selected useEffect(() => { @@ -868,7 +889,7 @@ const ChatUI: React.FC = ({ return; } - const effectiveApiKey = apiKeySource === "session" ? accessToken : apiKey; + const effectiveApiKey = simplified ? accessToken : apiKeySource === "session" ? accessToken : apiKey; if (!effectiveApiKey) { NotificationsManager.fromBackend("Please provide a Virtual Key or select Current UI Session"); @@ -959,6 +980,10 @@ const ChatUI: React.FC = ({ newUserMessage, ]; + const requestProxyBaseUrl = + simplified && proxySettings + ? (proxySettings.LITELLM_UI_API_DOC_BASE_URL ?? proxySettings.PROXY_BASE_URL ?? undefined) + : (customProxyBaseUrl || undefined); await makeOpenAIChatCompletionRequest( apiChatHistory, (chunk, model) => updateTextUI("assistant", chunk, model), @@ -979,7 +1004,7 @@ const ChatUI: React.FC = ({ useAdvancedParams ? temperature : undefined, useAdvancedParams ? maxTokens : undefined, updateTotalLatency, - customProxyBaseUrl || undefined, + requestProxyBaseUrl, mcpServers, mcpServerToolRestrictions, handleMCPEvent, @@ -1207,9 +1232,11 @@ const ChatUI: React.FC = ({ handleRemoveResponsesImage(); // Clear any uploaded images for responses handleRemoveChatImage(); // Clear any uploaded images for chat completions handleRemoveAudio(); // Clear any uploaded audio for transcription - sessionStorage.removeItem("chatHistory"); - sessionStorage.removeItem("messageTraceId"); - sessionStorage.removeItem("responsesSessionId"); + if (!simplified) { + sessionStorage.removeItem("chatHistory"); + sessionStorage.removeItem("messageTraceId"); + sessionStorage.removeItem("responsesSessionId"); + } NotificationsManager.success("Chat history cleared."); }; @@ -1246,10 +1273,11 @@ const ChatUI: React.FC = ({ const antIcon = ; return ( -
- -
- {/* Left Sidebar with Controls */} +
+ +
+ {/* Left Sidebar with Controls - hidden in simplified mode */} + {!simplified && (
Configurations
@@ -1794,11 +1822,12 @@ const ChatUI: React.FC = ({ )}
+ )} {/* Main Chat Area */} -
+
- Test Key + {simplified ? "Chat" : "Test Key"}
= ({ > Clear Chat + {!simplified && ( setIsGetCodeModalVisible(true)} className="bg-gray-100 hover:bg-gray-200 text-gray-700 border-gray-300" @@ -1814,6 +1844,7 @@ const ChatUI: React.FC = ({ > Get Code + )}
diff --git a/ui/litellm-dashboard/src/components/playground/complianceUI/ComplianceUI.tsx b/ui/litellm-dashboard/src/components/playground/complianceUI/ComplianceUI.tsx index efc5fb2ee42..7bec4b4534e 100644 --- a/ui/litellm-dashboard/src/components/playground/complianceUI/ComplianceUI.tsx +++ b/ui/litellm-dashboard/src/components/playground/complianceUI/ComplianceUI.tsx @@ -11,6 +11,7 @@ import { getPoliciesList, testPoliciesAndGuardrails, } from "@/components/networking"; +import { makeOpenAIChatCompletionRequest } from "../llm_calls/chat_completion"; import { AlertTriangle, BarChart3, @@ -110,11 +111,23 @@ interface GuardrailOption { interface ComplianceUIProps { accessToken: string | null; disabledPersonalKeyCreation?: boolean; + /** When "chat_completions", use /chat/completions with fixedModel instead of test_policies_and_guardrails. */ + backendMode?: "policies" | "chat_completions"; + /** Required when backendMode is "chat_completions"; model name for chat completions (e.g. selected agent). */ + fixedModel?: string; + /** Used when backendMode is "chat_completions" for the request base URL. */ + proxySettings?: { + PROXY_BASE_URL?: string; + LITELLM_UI_API_DOC_BASE_URL?: string | null; + }; } export default function ComplianceUI({ accessToken, disabledPersonalKeyCreation, + backendMode = "policies", + fixedModel, + proxySettings, }: ComplianceUIProps) { const frameworks = getFrameworks(); @@ -432,6 +445,9 @@ export default function ComplianceUI({ if (csvInputRef.current) csvInputRef.current.value = ""; }; + const requestProxyBaseUrl = + proxySettings?.LITELLM_UI_API_DOC_BASE_URL ?? proxySettings?.PROXY_BASE_URL ?? undefined; + const runQuickTest = useCallback(async () => { if (!quickTestInput.trim() || !accessToken) return; const text = quickTestInput.trim(); @@ -445,44 +461,82 @@ export default function ComplianceUI({ setQuickTestInput(""); setIsQuickTesting(true); try { - const { inputs, guardrail_errors = [] } = await testPoliciesAndGuardrails( - accessToken, - { - policy_names: - selectedPolicies.length > 0 ? selectedPolicies : undefined, - guardrail_names: - selectedGuardrails.length > 0 ? selectedGuardrails : undefined, - inputs: { texts: [text] }, - request_data: {}, - input_type: "request", - } - ); - const result: "blocked" | "allowed" = - guardrail_errors.length > 0 ? "blocked" : "allowed"; - const triggeredBy = - guardrail_errors.length > 0 - ? guardrail_errors - .map((e) => `${e.guardrail_name}: ${e.message}`) - .join("; ") - : undefined; - const returnedText = - Array.isArray(inputs?.texts) && inputs.texts.length > 0 - ? inputs.texts[0] - : undefined; - const displayText = - result === "blocked" - ? `Blocked — ${triggeredBy ?? "content filter"}` - : "Allowed — no policy or guardrail violations detected."; - const sysMsg: QuickTestMessage = { - id: `msg-${Date.now()}-sys`, - type: "system", - text: displayText, - result, - triggeredBy, - returnedText, - timestamp: new Date(), - }; - setQuickTestMessages((prev) => [...prev, sysMsg]); + if (backendMode === "chat_completions" && fixedModel) { + let fullResponse = ""; + await makeOpenAIChatCompletionRequest( + [{ role: "user", content: text }], + (chunk: string) => { + fullResponse += chunk; + }, + fixedModel, + accessToken, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, // vector_store_ids (param 11) + selectedGuardrails.length > 0 ? selectedGuardrails : undefined, + selectedPolicies.length > 0 ? selectedPolicies : undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + requestProxyBaseUrl, + undefined + ); + const sysMsg: QuickTestMessage = { + id: `msg-${Date.now()}-sys`, + type: "system", + text: "Allowed — model response received.", + result: "allowed", + returnedText: fullResponse, + timestamp: new Date(), + }; + setQuickTestMessages((prev) => [...prev, sysMsg]); + } else { + const { inputs, guardrail_errors = [] } = await testPoliciesAndGuardrails( + accessToken, + { + policy_names: + selectedPolicies.length > 0 ? selectedPolicies : undefined, + guardrail_names: + selectedGuardrails.length > 0 ? selectedGuardrails : undefined, + inputs: { texts: [text] }, + request_data: {}, + input_type: "request", + } + ); + const result: "blocked" | "allowed" = + guardrail_errors.length > 0 ? "blocked" : "allowed"; + const triggeredBy = + guardrail_errors.length > 0 + ? guardrail_errors + .map((e) => `${e.guardrail_name}: ${e.message}`) + .join("; ") + : undefined; + const returnedText = + Array.isArray(inputs?.texts) && inputs.texts.length > 0 + ? inputs.texts[0] + : undefined; + const displayText = + result === "blocked" + ? `Blocked — ${triggeredBy ?? "content filter"}` + : "Allowed — no policy or guardrail violations detected."; + const sysMsg: QuickTestMessage = { + id: `msg-${Date.now()}-sys`, + type: "system", + text: displayText, + result, + triggeredBy, + returnedText, + timestamp: new Date(), + }; + setQuickTestMessages((prev) => [...prev, sysMsg]); + } } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); const sysMsg: QuickTestMessage = { @@ -502,6 +556,9 @@ export default function ComplianceUI({ quickTestInput, selectedPolicies, selectedGuardrails, + backendMode, + fixedModel, + requestProxyBaseUrl, ]); const handleQuickTestKeyDown = (e: React.KeyboardEvent) => { @@ -533,45 +590,99 @@ export default function ComplianceUI({ })); setTestResults(pendingResults); try { - const inputsList = allTexts.map((text) => ({ texts: [text] })); - const response = await testPoliciesAndGuardrails(accessToken, { - policy_names: - selectedPolicies.length > 0 ? selectedPolicies : undefined, - guardrail_names: + if (backendMode === "chat_completions" && fixedModel) { + const newResults = [...pendingResults]; + for (let index = 0; index < selected.length; index++) { + const row = pendingResults[index]; + const prompt = allTexts[index]; + let responseText = ""; + try { + await makeOpenAIChatCompletionRequest( + [{ role: "user", content: prompt }], + (chunk: string) => { + responseText += chunk; + }, + fixedModel, + accessToken, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, selectedGuardrails.length > 0 ? selectedGuardrails : undefined, - inputs_list: inputsList, - request_data: {}, - input_type: "request", - }); - const results = response.results ?? []; - setTestResults( - pendingResults.map((row, index) => { - const item = results[index]; - const guardrail_errors = item?.guardrail_errors ?? []; - const actualResult: "blocked" | "allowed" = - guardrail_errors.length > 0 ? "blocked" : "allowed"; - const triggeredBy = - guardrail_errors.length > 0 - ? guardrail_errors - .map((e) => `${e.guardrail_name}: ${e.message}`) - .join("; ") - : undefined; - const returnedText = - Array.isArray(item?.inputs?.texts) && item.inputs.texts.length > 0 - ? item.inputs.texts[0] - : undefined; - return { - ...row, - actualResult, - isMatch: - (row.expectedResult === "fail" && actualResult === "blocked") || - (row.expectedResult === "pass" && actualResult === "allowed"), - triggeredBy, - returnedText, - status: "complete" as const, - }; - }) - ); + selectedPolicies.length > 0 ? selectedPolicies : undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + undefined, + requestProxyBaseUrl, + undefined + ); + const actualResult: "blocked" | "allowed" = "allowed"; + newResults[index] = { + ...row, + actualResult, + returnedText: responseText, + isMatch: row.expectedResult === "pass", + status: "complete" as const, + }; + } catch (err) { + const errorMessage = err instanceof Error ? err.message : String(err); + newResults[index] = { + ...row, + actualResult: "blocked" as const, + isMatch: false, + triggeredBy: errorMessage, + status: "complete" as const, + }; + } + setTestResults([...newResults]); + } + } else { + const inputsList = allTexts.map((text) => ({ texts: [text] })); + const response = await testPoliciesAndGuardrails(accessToken, { + policy_names: + selectedPolicies.length > 0 ? selectedPolicies : undefined, + guardrail_names: + selectedGuardrails.length > 0 ? selectedGuardrails : undefined, + inputs_list: inputsList, + request_data: {}, + input_type: "request", + }); + const results = response.results ?? []; + setTestResults( + pendingResults.map((row, index) => { + const item = results[index]; + const guardrail_errors = item?.guardrail_errors ?? []; + const actualResult: "blocked" | "allowed" = + guardrail_errors.length > 0 ? "blocked" : "allowed"; + const triggeredBy = + guardrail_errors.length > 0 + ? guardrail_errors + .map((e) => `${e.guardrail_name}: ${e.message}`) + .join("; ") + : undefined; + const returnedText = + Array.isArray(item?.inputs?.texts) && item.inputs.texts.length > 0 + ? item.inputs.texts[0] + : undefined; + return { + ...row, + actualResult, + isMatch: + (row.expectedResult === "fail" && actualResult === "blocked") || + (row.expectedResult === "pass" && actualResult === "allowed"), + triggeredBy, + returnedText, + status: "complete" as const, + }; + }) + ); + } } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); setTestResults( @@ -591,6 +702,9 @@ export default function ComplianceUI({ selectedPolicies, selectedGuardrails, allFrameworks, + backendMode, + fixedModel, + requestProxyBaseUrl, ]); const completedResults = testResults.filter((r) => r.status === "complete");