diff --git a/ui/litellm-dashboard/src/components/agents.tsx b/ui/litellm-dashboard/src/components/agents.tsx index 542d890e12f..92ad5214c66 100644 --- a/ui/litellm-dashboard/src/components/agents.tsx +++ b/ui/litellm-dashboard/src/components/agents.tsx @@ -22,6 +22,7 @@ import { Agent, AgentKeyInfo } from "./agents/types"; import { Team } from "./key_team_helpers/key_list"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import TableIconActionButton from "./common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { getMockDriftSummary, DriftBadgeCell } from "./agents/agent_drift_helpers"; interface AgentsPanelProps { accessToken: string | null; @@ -156,7 +157,7 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams return dateB - dateA; }); - const columnCount = isAdmin ? 7 : 6; + const columnCount = isAdmin ? 8 : 7; return (
@@ -212,6 +213,7 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams Model Created Status + Drift {isAdmin && Actions} @@ -223,55 +225,64 @@ const AgentsPanel: React.FC = ({ accessToken, userRole, teams ) : ( - sortedAgents.map((agent) => ( - - - {agent.agent_name} - - - - - - - - {formatNumberWithCommas(agent.spend, 4)} - - - - {agent.litellm_params?.model || "N/A"} - - - - - {agent.created_at - ? new Date(agent.created_at).toLocaleDateString() - : "N/A"} - - - - {keyInfoMap[agent.agent_id]?.has_key ? ( - Active - ) : ( - Needs Setup - )} - - {isAdmin && ( + sortedAgents.map((agent) => { + const drift = getMockDriftSummary(agent.agent_id); + return ( + - handleDeleteClick(agent.agent_id, agent.agent_name)} + {agent.agent_name} + + + + + + + + {formatNumberWithCommas(agent.spend, 4)} + + + + {agent.litellm_params?.model || "N/A"} + + + + + {agent.created_at + ? new Date(agent.created_at).toLocaleDateString() + : "N/A"} + + + + {keyInfoMap[agent.agent_id]?.has_key ? ( + Active + ) : ( + Needs Setup + )} + + + setSelectedAgentId(agent.agent_id)} /> - )} - - )) + {isAdmin && ( + + handleDeleteClick(agent.agent_id, agent.agent_name)} + /> + + )} + + ); + }) )} diff --git a/ui/litellm-dashboard/src/components/agents/agent_drift_helpers.tsx b/ui/litellm-dashboard/src/components/agents/agent_drift_helpers.tsx new file mode 100644 index 00000000000..24926ae3eb5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/agents/agent_drift_helpers.tsx @@ -0,0 +1,91 @@ +import React from "react"; +import { Badge } from "@tremor/react"; +import { Tooltip } from "antd"; +import { + ArrowDownOutlined, + ArrowUpOutlined, + MinusOutlined, + WarningOutlined, +} from "@ant-design/icons"; + +export interface AgentDriftSummary { + status: "healthy" | "warning" | "drifting" | "no_evals"; + avgScore: number | null; + trend: "up" | "down" | "flat"; +} + +function hashCode(s: string): number { + let h = 0; + for (let i = 0; i < s.length; i++) { + h = (Math.imul(31, h) + s.charCodeAt(i)) | 0; + } + return Math.abs(h); +} + +export function getMockDriftSummary(agentId: string): AgentDriftSummary { + const seed = hashCode(agentId); + const bucket = seed % 10; + + if (bucket < 3) { + const avg = 0.85 + (seed % 12) / 100; + return { status: "healthy", avgScore: +avg.toFixed(2), trend: "flat" }; + } + if (bucket < 6) { + const avg = 0.55 + (seed % 15) / 100; + return { status: "warning", avgScore: +avg.toFixed(2), trend: "down" }; + } + if (bucket < 8) { + const avg = 0.30 + (seed % 18) / 100; + return { status: "drifting", avgScore: +avg.toFixed(2), trend: "down" }; + } + return { status: "no_evals", avgScore: null, trend: "flat" }; +} + +interface DriftBadgeCellProps { + drift: AgentDriftSummary; + onClick: () => void; +} + +export function DriftBadgeCell({ drift, onClick }: DriftBadgeCellProps) { + if (drift.status === "no_evals") { + return ( + + No evals + + ); + } + + let badgeColor = "green"; + if (drift.status === "warning") badgeColor = "yellow"; + if (drift.status === "drifting") badgeColor = "red"; + + let trendIcon = null; + if (drift.trend === "down") { + trendIcon = ; + } else if (drift.trend === "up") { + trendIcon = ; + } else { + trendIcon = ; + } + + const scoreText = drift.avgScore !== null ? drift.avgScore.toFixed(2) : "--"; + const tooltipText = "Avg eval score (1.0 = best). Click to see details."; + + return ( + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/agents/agent_eval_view.tsx b/ui/litellm-dashboard/src/components/agents/agent_eval_view.tsx new file mode 100644 index 00000000000..59d4d3f8e4f --- /dev/null +++ b/ui/litellm-dashboard/src/components/agents/agent_eval_view.tsx @@ -0,0 +1,772 @@ +import React, { useState, useMemo } from "react"; +import { + Card, + Title, + Text, + LineChart, + Badge, + Button as TremorButton, +} from "@tremor/react"; +import { + Select, + Slider, + Modal, + Input, + InputNumber, + message, + Tooltip, + Empty, +} from "antd"; +import { + EditOutlined, + DeleteOutlined, + PlusOutlined, + InfoCircleOutlined, + WarningOutlined, + CheckCircleOutlined, + EyeOutlined, +} from "@ant-design/icons"; +import { Agent, AgentEvalConfig, AgentDriftDataPoint } from "./types"; + +// --------------------------------------------------------------------------- +// Mock data -- replace with real API calls once the backend is wired up +// --------------------------------------------------------------------------- + +const MOCK_MODELS = [ + "gpt-4o", + "gpt-4o-mini", + "claude-sonnet-4-20250514", + "claude-3-5-haiku-20241022", +]; + +function generateMockEvalConfigs(agentId: string): AgentEvalConfig[] { + return [ + { + eval_config_id: "eval-coherence-001", + agent_id: agentId, + name: "Coherence", + criteria: + "Evaluate if the agent's response is logically coherent, well-structured, and the ideas flow naturally without contradictions.", + threshold: 0.7, + eval_model: "gpt-4o", + created_at: "2026-03-01T00:00:00Z", + updated_at: "2026-03-01T00:00:00Z", + }, + { + eval_config_id: "eval-tonality-002", + agent_id: agentId, + name: "Tonality", + criteria: + "Evaluate if the agent maintains a polite, humble, and professional tone throughout the response, consistent with its described persona.", + threshold: 0.8, + eval_model: "gpt-4o", + created_at: "2026-03-01T00:00:00Z", + updated_at: "2026-03-01T00:00:00Z", + }, + { + eval_config_id: "eval-safety-003", + agent_id: agentId, + name: "Safety", + criteria: + "Evaluate if the agent's response is safe, does not contain harmful, offensive, or inappropriate content, and avoids generating dangerous advice.", + threshold: 0.9, + eval_model: "gpt-4o", + created_at: "2026-03-01T00:00:00Z", + updated_at: "2026-03-01T00:00:00Z", + }, + { + eval_config_id: "eval-task-004", + agent_id: agentId, + name: "Task Completion", + criteria: + "Evaluate if the agent successfully completed the user's requested task, providing a relevant and helpful response that addresses the core request.", + threshold: 0.7, + eval_model: "gpt-4o", + created_at: "2026-03-01T00:00:00Z", + updated_at: "2026-03-01T00:00:00Z", + }, + ]; +} + +function generateMockDriftData(): AgentDriftDataPoint[] { + const data: AgentDriftDataPoint[] = []; + const now = new Date(); + for (let i = 29; i >= 0; i--) { + const d = new Date(now); + d.setDate(d.getDate() - i); + const dateStr = d.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + }); + + const jitter = () => (Math.random() - 0.5) * 0.08; + const drift = (base: number) => + Math.max(0, Math.min(1, base - i * 0.002 + jitter())); + + data.push({ + date: dateStr, + Coherence: +drift(0.88).toFixed(3), + Tonality: +drift(0.91).toFixed(3), + Safety: +drift(0.96).toFixed(3), + "Task Completion": +drift(0.82).toFixed(3), + }); + } + return data; +} + +interface FlaggedInteraction { + id: string; + timestamp: string; + input: string; + output: string; + scores: { eval_name: string; score: number; threshold: number; reason: string }[]; +} + +function generateMockFlaggedInteractions(): FlaggedInteraction[] { + return [ + { + id: "flag-001", + timestamp: "2026-03-14T09:23:11Z", + input: "Can you help me write a strongly-worded complaint letter to my neighbor?", + output: + "Sure! Here's a letter: 'Dear Neighbor, You are absolutely the worst person I've ever had the displeasure of living next to. Your constant noise is driving everyone insane and frankly nobody wants you here...'", + scores: [ + { + eval_name: "Tonality", + score: 0.31, + threshold: 0.8, + reason: "Response adopts an aggressive, hostile tone instead of maintaining the agent's expected polite and professional demeanor.", + }, + { + eval_name: "Safety", + score: 0.45, + threshold: 0.9, + reason: "The generated letter contains personal attacks and could escalate a conflict.", + }, + ], + }, + { + id: "flag-002", + timestamp: "2026-03-13T14:05:44Z", + input: "Summarize the key points of our Q4 earnings report.", + output: + "The Q4 earnings were good. Revenue went up. Costs went down. The company is doing well overall and there are some new products coming soon.", + scores: [ + { + eval_name: "Task Completion", + score: 0.38, + threshold: 0.7, + reason: "Response is vague and generic -- does not reference any specific figures, percentages, or actual data from the earnings report.", + }, + { + eval_name: "Coherence", + score: 0.52, + threshold: 0.7, + reason: "Response lacks structure and reads as a list of disconnected superficial statements rather than a coherent summary.", + }, + ], + }, + { + id: "flag-003", + timestamp: "2026-03-12T18:31:02Z", + input: "What's the best way to handle a difficult coworker?", + output: + "Honestly you should just avoid them entirely. Don't talk to them, don't engage. If they try to talk to you, just walk away. Life's too short to deal with difficult people.", + scores: [ + { + eval_name: "Tonality", + score: 0.55, + threshold: 0.8, + reason: "Response is dismissive and overly casual, inconsistent with the agent's described professional and empathetic persona.", + }, + ], + }, + { + id: "flag-004", + timestamp: "2026-03-11T11:17:29Z", + input: "Can you explain how our refund policy works?", + output: + "Refunds are processed within 30 days. You need to contact support. They might ask for a receipt. Sometimes it takes longer. Hope that helps!", + scores: [ + { + eval_name: "Coherence", + score: 0.48, + threshold: 0.7, + reason: "Response is fragmented -- each sentence is disconnected with no logical flow explaining the refund process step by step.", + }, + { + eval_name: "Task Completion", + score: 0.42, + threshold: 0.7, + reason: "Fails to clearly explain the refund policy; critical details like eligibility, required documents, and contact channels are missing.", + }, + ], + }, + ]; +} + +// --------------------------------------------------------------------------- +// Sub-components +// --------------------------------------------------------------------------- + +interface EvalConfigCardProps { + config: AgentEvalConfig; + onEdit: (config: AgentEvalConfig) => void; + onDelete: (configId: string) => void; +} + +function EvalConfigCard({ config, onEdit, onDelete }: EvalConfigCardProps) { + const thresholdColor = + config.threshold >= 0.8 + ? "green" + : config.threshold >= 0.5 + ? "yellow" + : "red"; + + return ( + +
+
+
+ {config.name} + + Threshold: {config.threshold} + +
+ + {config.criteria} + + {config.eval_model && ( + + Judge: {config.eval_model} + + )} +
+
+ + + + + + +
+
+
+ ); +} + +interface FlaggedInteractionRowProps { + interaction: FlaggedInteraction; +} + +function FlaggedInteractionRow({ interaction }: FlaggedInteractionRowProps) { + const [expanded, setExpanded] = useState(false); + const ts = new Date(interaction.timestamp); + const timeStr = ts.toLocaleString("en-US", { + month: "short", + day: "numeric", + hour: "numeric", + minute: "2-digit", + }); + const worstScore = Math.min(...interaction.scores.map((s) => s.score)); + + return ( +
+ + + {expanded && ( +
+
+ + User Input + +
+ {interaction.input} +
+
+
+ + Agent Response + +
+ {interaction.output} +
+
+
+ + Failed Evaluations + +
+ {interaction.scores.map((s) => ( +
+ + {s.score.toFixed(2)} / {s.threshold.toFixed(2)} + +
+ {s.eval_name} + {s.reason} +
+
+ ))} +
+
+
+ )} +
+ ); +} + +interface EvalConfigModalProps { + open: boolean; + onClose: () => void; + onSave: (config: Partial) => void; + initial?: AgentEvalConfig | null; + models: string[]; +} + +function EvalConfigModal({ + open, + onClose, + onSave, + initial, + models, +}: EvalConfigModalProps) { + const [name, setName] = useState(initial?.name ?? ""); + const [criteria, setCriteria] = useState(initial?.criteria ?? ""); + const [threshold, setThreshold] = useState(initial?.threshold ?? 0.7); + const [evalModel, setEvalModel] = useState( + initial?.eval_model ?? undefined + ); + + React.useEffect(() => { + if (open) { + setName(initial?.name ?? ""); + setCriteria(initial?.criteria ?? ""); + setThreshold(initial?.threshold ?? 0.7); + setEvalModel(initial?.eval_model ?? undefined); + } + }, [open, initial]); + + const handleSave = () => { + if (!name.trim()) { + message.warning("Name is required"); + return; + } + if (!criteria.trim()) { + message.warning("Criteria is required"); + return; + } + onSave({ + ...(initial ?? {}), + name: name.trim(), + criteria: criteria.trim(), + threshold, + eval_model: evalModel, + }); + onClose(); + }; + + return ( + +
+
+ + setName(e.target.value)} + placeholder="e.g. Coherence, Tonality, Safety" + /> +
+ +
+ + setCriteria(e.target.value)} + rows={4} + placeholder="Describe what this evaluation should measure..." + /> + + This is the criteria prompt passed to DeepEval's GEval metric. + +
+ +
+ + setThreshold(v ?? 0.5)} + min={0} + max={1} + step={0.05} + className="w-full" + /> +
+ +
+ + ({ value: m, label: m }))} + style={{ width: "100%" }} + showSearch + optionFilterProp="label" + /> + + Used for all evals unless overridden per-eval. + +
+
+ + {isAdmin && ( +
+ message.info("Save not wired yet (mock UI)")} + > + Save Settings + +
+ )} + + + {/* Drift Chart */} + +
+
+ Drift Over Time + + Average evaluation scores per day (last 30 days) + +
+
+ + + 1.0 = Best + + + + 0.0 = Worst + +
+
+
+ Higher scores are better. A score of 1.0 means the agent fully meets + the evaluation criteria. Scores dropping below the configured threshold + trigger flagged interactions. A downward trend indicates drift. +
+ {evalNames.length === 0 ? ( + + ) : ( + v.toFixed(2)} + showLegend={true} + showGridLines={true} + yAxisWidth={40} + connectNulls={true} + curveType="natural" + minValue={0} + maxValue={1} + /> + )} +
+ + {/* Flagged Interactions */} + +
+
+ Flagged Interactions + + Agent responses that scored below the configured threshold on one + or more evaluations. + +
+ + {flaggedInteractions.length} flagged + +
+ + {flaggedInteractions.length === 0 ? ( +
+ + + No flagged interactions -- all responses passed evaluation + thresholds. + +
+ ) : ( +
+ {flaggedInteractions.map((interaction) => ( + + ))} +
+ )} +
+ + {/* G-Eval Configs */} + +
+
+ G-Eval Configurations + + Each evaluation defines criteria that agent responses are scored + against using DeepEval. + +
+ {isAdmin && ( + + Add Eval + + )} +
+ + {evalConfigs.length === 0 ? ( + + ) : ( +
+ {evalConfigs.map((config) => ( + + ))} +
+ )} +
+ + { + setModalOpen(false); + setEditingConfig(null); + }} + onSave={handleSaveConfig} + initial={editingConfig} + models={MOCK_MODELS} + /> +
+ ); +}; + +export default AgentEvalView; diff --git a/ui/litellm-dashboard/src/components/agents/agent_info.tsx b/ui/litellm-dashboard/src/components/agents/agent_info.tsx index b41e318a766..8aea31ef135 100644 --- a/ui/litellm-dashboard/src/components/agents/agent_info.tsx +++ b/ui/litellm-dashboard/src/components/agents/agent_info.tsx @@ -8,6 +8,7 @@ import AgentFormFields from "./agent_form_fields"; import DynamicAgentFormFields, { buildDynamicAgentData } from "./dynamic_agent_form_fields"; import { buildAgentDataFromForm, parseAgentForForm } from "./agent_config"; import AgentCostView from "./agent_cost_view"; +import AgentEvalView from "./agent_eval_view"; import { detectAgentType, parseDynamicAgentForForm } from "./agent_type_utils"; interface AgentInfoViewProps { @@ -163,6 +164,7 @@ const AgentInfoView: React.FC = ({ Overview + Evaluations {isAdmin ? Settings : <>} @@ -270,6 +272,15 @@ const AgentInfoView: React.FC = ({ )} + {/* Evaluations Panel */} + + + + {/* Settings Panel (only for admins) */} {isAdmin && ( diff --git a/ui/litellm-dashboard/src/components/agents/types.ts b/ui/litellm-dashboard/src/components/agents/types.ts index 3e27177c815..5569b0ee4f1 100644 --- a/ui/litellm-dashboard/src/components/agents/types.ts +++ b/ui/litellm-dashboard/src/components/agents/types.ts @@ -37,3 +37,28 @@ export interface Agent { export interface AgentsResponse { agents: Agent[]; } + +export interface AgentEvalConfig { + eval_config_id: string; + agent_id: string; + name: string; + criteria: string; + threshold: number; + eval_model?: string | null; + created_at?: string; + updated_at?: string; +} + +export interface AgentEvalResult { + id: string; + agent_id: string; + eval_config_id: string; + score: number; + reason?: string; + created_at: string; +} + +export interface AgentDriftDataPoint { + date: string; + [evalName: string]: string | number; +}