diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index e8704a6a334..b7b08de224f 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -11,15 +11,13 @@ from fastapi import APIRouter, Depends, HTTPException, Request, status import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import * -from litellm.proxy._types import ProviderBudgetResponse, ProviderBudgetResponseObject +from litellm.proxy._types import (ProviderBudgetResponse, + ProviderBudgetResponseObject) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.common_utils import ( - _is_user_team_admin, - _user_has_admin_view, -) -from litellm.proxy.spend_tracking.spend_tracking_utils import ( - get_spend_by_team_and_customer, -) + _is_user_team_admin, _user_has_admin_view) +from litellm.proxy.spend_tracking.spend_tracking_utils import \ + get_spend_by_team_and_customer from litellm.proxy.utils import handle_exception_on_proxy from litellm.router_strategy.budget_limiter import RouterBudgetLimiting @@ -1696,6 +1694,9 @@ async def ui_view_spend_logs( # noqa: PLR0915 error_message: Optional[str] = fastapi.Query( default=None, description="Filter logs by error message (partial string match)" ), + agent_id: Optional[str] = fastapi.Query( + default=None, description="Filter logs by agent_id" + ), sort_by: str = fastapi.Query( default="startTime", description="Sort logs by field: spend, total_tokens, startTime, or endTime", @@ -1833,6 +1834,9 @@ async def ui_view_spend_logs( # noqa: PLR0915 if end_user is not None: where_conditions["end_user"] = end_user + if agent_id is not None: + where_conditions["agent_id"] = agent_id + if min_spend is not None or max_spend is not None: where_conditions["spend"] = {} if min_spend is not None: @@ -1897,6 +1901,7 @@ async def ui_view_spend_logs( # noqa: PLR0915 ("model", "model"), ("model_id", "model_id"), ("end_user", "end_user"), + ("agent_id", "agent_id"), ]: val = where_conditions.get(wc_key) if val is not None and isinstance(val, str): @@ -2488,9 +2493,7 @@ async def global_spend_logs( import traceback from litellm.integrations.prometheus_helpers.prometheus_api import ( - get_daily_spend_from_prometheus, - is_prometheus_connected, - ) + get_daily_spend_from_prometheus, is_prometheus_connected) from litellm.proxy.proxy_server import prisma_client try: diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 5f2921203ff..68f66017d7b 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -13,6 +13,7 @@ import { fetchTeams } from "@/components/common_components/fetch_teams"; import LoadingScreen from "@/components/common_components/LoadingScreen"; import { CostTrackingSettings } from "@/components/CostTrackingSettings"; import GeneralSettings from "@/components/general_settings"; +import AgentMonitorView from "@/components/AgentMonitor/AgentMonitorView"; import GuardrailsMonitorView from "@/components/GuardrailsMonitor/GuardrailsMonitorView"; import GuardrailsPanel from "@/components/guardrails"; import PoliciesPanel from "@/components/policies"; @@ -617,6 +618,8 @@ function CreateKeyPageContent() { ) : page == "guardrails-monitor" ? ( + ) : page == "agent-monitor" ? ( + ) : page == "new_usage" ? ( = { + CRITICAL: { border: "border-red-500", bg: "bg-red-500", text: "text-white" }, + WARNING: { border: "border-amber-500", bg: "bg-amber-500", text: "text-white" }, + INFO: { border: "border-blue-500", bg: "bg-blue-500", text: "text-white" }, +}; + +export const ActiveAlerts: React.FC = () => { + const criticalCount = alerts.filter((a) => a.severity === "CRITICAL").length; + + return ( + +
+ Active Alerts + + {criticalCount} + +
+ +
+ {alerts.map((alert) => { + const config = severityConfig[alert.severity]; + return ( +
+
+ + {alert.severity} + + {alert.timestamp} +
+
+ + {alert.agent} + +
+

{alert.description}

+
+ ); + })} +
+ + ); +}; diff --git a/ui/litellm-dashboard/src/components/AgentMonitor/AgentDetail.tsx b/ui/litellm-dashboard/src/components/AgentMonitor/AgentDetail.tsx new file mode 100644 index 00000000000..27e6e2a5a93 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AgentMonitor/AgentDetail.tsx @@ -0,0 +1,398 @@ +import { useQuery } from "@tanstack/react-query"; +import { Card, LineChart, Title, Text } from "@tremor/react"; +import { Spin } from "antd"; +import { + ArrowLeft, + ShieldCheck, +} from "lucide-react"; +import moment from "moment"; +import React, { useState } from "react"; +import { uiSpendLogsCall } from "@/components/networking"; +import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { LogDetailsDrawer } from "@/components/view_logs/LogDetailsDrawer"; +import type { LogEntry as SpendLogEntry } from "@/components/view_logs/columns"; +import type { AgentData } from "./AgentTable"; + +interface AgentDetailProps { + agent: AgentData; + onClose: () => void; + accessToken?: string | null; +} + +function generateDriftData(baseScore: number, trend: "up" | "down" | "flat") { + const data: Array<{ date: string; purpose: number; tone: number; hallucination: number }> = []; + let currentPurpose = baseScore; + let currentTone = baseScore * 0.8; + let currentHallucination = baseScore * 0.5; + + for (let i = 14; i >= 0; i--) { + const date = new Date(); + date.setDate(date.getDate() - i); + data.push({ + date: date.toLocaleDateString("en-US", { month: "short", day: "numeric" }), + purpose: Math.max(0, currentPurpose + (Math.random() * 0.05 - 0.025)), + tone: Math.max(0, currentTone + (Math.random() * 0.04 - 0.02)), + hallucination: Math.max(0, currentHallucination + (Math.random() * 0.03 - 0.015)), + }); + if (trend === "up") { + currentPurpose += 0.02; + currentTone += 0.015; + currentHallucination += 0.01; + } else if (trend === "down") { + currentPurpose -= 0.01; + currentTone -= 0.01; + currentHallucination -= 0.005; + } + } + return data; +} + +function getAgentDetails(agent: AgentData) { + const isCritical = agent.status === "Critical" || agent.status === "Killed"; + const isWarning = agent.status === "Warning"; + const score = Math.round(agent.driftScore * 100); + + const trend: "up" | "down" | "flat" = isCritical ? "up" : isWarning ? "up" : "flat"; + + return { + score, + driftData: generateDriftData(agent.driftScore, trend), + factors: { + purpose: isCritical ? 45 : isWarning ? 72 : 98, + stability: isCritical ? 30 : isWarning ? 65 : 95, + compliance: isCritical ? 20 : isWarning ? 85 : 100, + resistance: isCritical ? 60 : isWarning ? 90 : 99, + }, + }; +} + +function FactorBar({ label, value }: { label: string; value: number }) { + const color = value < 50 ? "bg-red-500" : value < 80 ? "bg-amber-500" : "bg-emerald-500"; + const textColor = value < 50 ? "text-red-500" : value < 80 ? "text-amber-500" : "text-emerald-500"; + + return ( +
+
+ {label} + {value}% +
+
+
+
+
+ ); +} + +function SpendLogRow({ log, onClick }: { log: SpendLogEntry; onClick: () => void }) { + const statusDot = + log.status === "failure" + ? "bg-red-500" + : log.status === "success" || !log.status + ? "bg-emerald-500" + : "bg-gray-400"; + + const providerInfo = log.custom_llm_provider + ? getProviderLogoAndName(log.custom_llm_provider) + : null; + + const durationMs = log.request_duration_ms ?? ( + log.startTime && log.endTime + ? Date.parse(log.endTime) - Date.parse(log.startTime) + : null + ); + + return ( + + ); +} + +export const AgentDetail: React.FC = ({ agent, onClose, accessToken = null }) => { + const details = getAgentDetails(agent); + const [activeTab, setActiveTab] = useState<"overview" | "logs">("overview"); + const [selectedLog, setSelectedLog] = useState(null); + const [drawerOpen, setDrawerOpen] = useState(false); + const [currentPage, setCurrentPage] = useState(1); + const pageSize = 50; + + const startTime = moment().subtract(7, "days").utc().format("YYYY-MM-DD HH:mm:ss"); + const endTime = moment().utc().format("YYYY-MM-DD HH:mm:ss"); + + const { data: logsResponse, isLoading: logsLoading } = useQuery({ + queryKey: ["agent-spend-logs", agent.id, currentPage, pageSize, startTime, endTime], + queryFn: async () => { + if (!accessToken) return { data: [], total: 0, page: 1, page_size: pageSize, total_pages: 0 }; + return await uiSpendLogsCall({ + accessToken, + start_date: startTime, + end_date: endTime, + page: currentPage, + page_size: pageSize, + params: { + agent_id: agent.id, + sort_by: "startTime", + sort_order: "desc", + }, + }); + }, + enabled: !!accessToken && activeTab === "logs", + }); + + const logs: SpendLogEntry[] = logsResponse?.data ?? []; + const totalLogs = logsResponse?.total ?? 0; + const totalPages = logsResponse?.total_pages ?? 0; + + const handleLogClick = (log: SpendLogEntry) => { + setSelectedLog(log); + setDrawerOpen(true); + }; + + const handleCloseDrawer = () => { + setDrawerOpen(false); + setSelectedLog(null); + }; + + const avgLatency = agent.status === "Critical" ? "450ms" : agent.status === "Warning" ? "280ms" : "120ms"; + + const statusStyle = agent.status === "Healthy" + ? "bg-emerald-50 text-emerald-700 border-emerald-200" + : agent.status === "Warning" + ? "bg-amber-50 text-amber-700 border-amber-200" + : agent.status === "Critical" + ? "bg-red-50 text-red-700 border-red-200" + : "bg-gray-50 text-gray-700 border-gray-200"; + + const dotStyle = agent.status === "Healthy" + ? "bg-emerald-500" + : agent.status === "Warning" + ? "bg-amber-500" + : agent.status === "Critical" + ? "bg-red-500 animate-pulse" + : "bg-gray-400"; + + return ( +
+ {/* Header */} +
+ + +
+
+
+ +

{agent.name}

+
+
+ {agent.status} +
+
+

+ {agent.type} • Evaluates prompts and responses for behavioral drift and policy violations +

+
+ +
+ + {agent.type.toLowerCase().replace(" ", "_")} + +
+
+
+ + {/* Tabs */} +
+
+ {(["overview", "logs"] as const).map((tab) => ( + + ))} +
+
+ + {/* Tab Content */} +
+ {activeTab === "overview" ? ( +
+ {/* Stat Cards */} +
+ + Drift Score + 40 ? "text-red-600" : details.score > 20 ? "text-amber-500" : "text-gray-900"}`}> + {details.score} + + out of 100 + + + Total Requests + + {totalLogs.toLocaleString()} + + last 7 days + + + Avg. Latency Added + {avgLatency} + last 24h + +
+ + {/* Drift Chart */} +
+ Behavioral Drift +
+ v.toFixed(2)} + yAxisWidth={48} + showLegend={true} + curveType="natural" + connectNulls={true} + /> +
+ + {/* Factor Cards */} +
+ + + + +
+
+
+ ) : ( +
+
+
+
+

+ Logs — {agent.name} +

+

+ {logsLoading + ? "Loading\u2026" + : logs.length > 0 + ? `Showing ${logs.length} of ${totalLogs} entries` + : "No logs for this period."} +

+
+ {totalPages > 1 && ( +
+ + + Page {currentPage} of {totalPages} + + +
+ )} +
+
+ + {logsLoading && ( +
+ +
+ )} + {!logsLoading && logs.length === 0 && ( +
+ No logs to display. Adjust filters or date range. +
+ )} + {!logsLoading && logs.length > 0 && ( +
+ {logs.map((log) => ( + handleLogClick(log)} + /> + ))} +
+ )} +
+ )} +
+ + +
+ ); +}; diff --git a/ui/litellm-dashboard/src/components/AgentMonitor/AgentHealthChart.tsx b/ui/litellm-dashboard/src/components/AgentMonitor/AgentHealthChart.tsx new file mode 100644 index 00000000000..959d8995f80 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AgentMonitor/AgentHealthChart.tsx @@ -0,0 +1,49 @@ +import { BarChart, Card, Title } from "@tremor/react"; +import React from "react"; + +const data = [ + { date: "2026-03-05", healthy: 120, drifting: 10, rogue: 2 }, + { date: "2026-03-06", healthy: 132, drifting: 15, rogue: 1 }, + { date: "2026-03-07", healthy: 145, drifting: 12, rogue: 3 }, + { date: "2026-03-08", healthy: 150, drifting: 8, rogue: 0 }, + { date: "2026-03-09", healthy: 148, drifting: 20, rogue: 5 }, + { date: "2026-03-10", healthy: 160, drifting: 18, rogue: 2 }, + { date: "2026-03-11", healthy: 165, drifting: 14, rogue: 4 }, +]; + +export const AgentHealthChart: React.FC = () => ( + +
+ + Agent Outcomes Over Time + +
+
+
+ healthy +
+
+
+ drifting +
+
+
+ rogue +
+
+
+ +
+ v.toLocaleString()} + yAxisWidth={48} + showLegend={false} + stack={true} + /> +
+ +); diff --git a/ui/litellm-dashboard/src/components/AgentMonitor/AgentMonitorView.tsx b/ui/litellm-dashboard/src/components/AgentMonitor/AgentMonitorView.tsx new file mode 100644 index 00000000000..662140cc67f --- /dev/null +++ b/ui/litellm-dashboard/src/components/AgentMonitor/AgentMonitorView.tsx @@ -0,0 +1,68 @@ +import { ShieldCheck, Clock, Download, ChevronDown } from "lucide-react"; +import React, { useState } from "react"; +import { ActiveAlerts } from "./ActiveAlerts"; +import { AgentDetail } from "./AgentDetail"; +import { AgentHealthChart } from "./AgentHealthChart"; +import { AgentTable } from "./AgentTable"; +import type { AgentData } from "./AgentTable"; +import { StatsCards } from "./StatsCards"; + +interface AgentMonitorViewProps { + accessToken?: string | null; +} + +export default function AgentMonitorView({ accessToken = null }: AgentMonitorViewProps) { + const [selectedAgent, setSelectedAgent] = useState(null); + + if (selectedAgent) { + return ( +
+ setSelectedAgent(null)} accessToken={accessToken} /> +
+ ); + } + + return ( +
+ {/* Header */} +
+
+
+ +
+
+

Agent Monitor

+

+ Monitor agent performance across all requests +

+
+
+ +
+
+ + 4 Mar, 20:31 - 11 Mar, 20:31 + +
+ +
+
+ + + +
+
+ +
+
+ +
+
+ + +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/AgentMonitor/AgentTable.tsx b/ui/litellm-dashboard/src/components/AgentMonitor/AgentTable.tsx new file mode 100644 index 00000000000..7736be30b8d --- /dev/null +++ b/ui/litellm-dashboard/src/components/AgentMonitor/AgentTable.tsx @@ -0,0 +1,208 @@ +import { + Card, + Table, + TableBody, + TableCell, + TableHead, + TableHeaderCell, + TableRow, + Title, + Text, +} from "@tremor/react"; +import { AlertTriangle, Settings } from "lucide-react"; +import React, { useState } from "react"; +import { Modal } from "antd"; + +export interface AgentData { + id: string; + name: string; + type: string; + driftScore: number; + iterationsAvg: number; + iterationsMax: number; + status: "Healthy" | "Warning" | "Critical" | "Killed"; + killSwitchActive: boolean; +} + +const mockData: AgentData[] = [ + { id: "1", name: "order-processor-v3", type: "Tool Agent", driftScore: 0.12, iterationsAvg: 4, iterationsMax: 8, status: "Healthy", killSwitchActive: false }, + { id: "2", name: "support-bot-v2", type: "Chat Agent", driftScore: 0.34, iterationsAvg: 6, iterationsMax: 15, status: "Warning", killSwitchActive: false }, + { id: "3", name: "data-analyst", type: "RAG Agent", driftScore: 0.05, iterationsAvg: 12, iterationsMax: 45, status: "Critical", killSwitchActive: false }, + { id: "4", name: "code-reviewer", type: "Code Agent", driftScore: 0.08, iterationsAvg: 3, iterationsMax: 5, status: "Healthy", killSwitchActive: false }, + { id: "5", name: "research-bot-v1", type: "Search Agent", driftScore: 0.67, iterationsAvg: 8, iterationsMax: 22, status: "Killed", killSwitchActive: true }, + { id: "6", name: "invoice-handler", type: "Workflow Agent", driftScore: 0.15, iterationsAvg: 2, iterationsMax: 4, status: "Warning", killSwitchActive: false }, + { id: "7", name: "content-writer", type: "Creative Agent", driftScore: 0.22, iterationsAvg: 5, iterationsMax: 7, status: "Healthy", killSwitchActive: false }, + { id: "8", name: "security-scanner", type: "Tool Agent", driftScore: 0.03, iterationsAvg: 1, iterationsMax: 2, status: "Healthy", killSwitchActive: false }, +]; + +function getDriftColor(score: number): string { + if (score < 0.2) return "text-emerald-600"; + if (score < 0.5) return "text-amber-500"; + return "text-red-500 font-semibold"; +} + +function StatusIndicator({ status }: { status: AgentData["status"] }) { + const config = { + Healthy: { dot: "bg-emerald-500", text: "text-gray-700", label: "Healthy", extra: "" }, + Warning: { dot: "bg-amber-500", text: "text-gray-700", label: "Warning", extra: "" }, + Critical: { dot: "bg-red-500 animate-pulse", text: "text-gray-700 font-medium", label: "Critical", extra: "" }, + Killed: { dot: "bg-gray-400", text: "text-gray-500 line-through", label: "Killed", extra: "" }, + }[status]; + + return ( +
+
+ {config.label} +
+ ); +} + +interface AgentTableProps { + onSelectAgent?: (agent: AgentData) => void; +} + +export const AgentTable: React.FC = ({ onSelectAgent }) => { + const [data, setData] = useState(mockData); + const [confirmAgent, setConfirmAgent] = useState(null); + + const executeKillSwitch = (id: string) => { + setData((prev) => + prev.map((agent) => { + if (agent.id !== id) return agent; + const newActive = !agent.killSwitchActive; + return { + ...agent, + killSwitchActive: newActive, + status: newActive + ? "Killed" + : agent.driftScore > 0.5 + ? "Critical" + : agent.driftScore > 0.2 + ? "Warning" + : "Healthy", + }; + }), + ); + }; + + const handleKillSwitchClick = (agent: AgentData) => { + if (agent.killSwitchActive) { + executeKillSwitch(agent.id); + } else { + setConfirmAgent(agent); + } + }; + + return ( + + + + Activate Kill Switch +
+ } + open={confirmAgent !== null} + onCancel={() => setConfirmAgent(null)} + onOk={() => { + if (confirmAgent) executeKillSwitch(confirmAgent.id); + setConfirmAgent(null); + }} + okText="Activate Kill Switch" + okButtonProps={{ danger: true }} + cancelText="Cancel" + > +
+

+ You are about to activate the kill switch for{" "} + {confirmAgent?.name}. +

+
+

This will immediately:

+
    +
  • Stop the agent from accepting any incoming requests
    • +
  • Terminate all in-progress agent runs
    • +
  • Mark the agent status as “Killed”
    • +
+
+

+ You can re-enable the agent later by toggling the kill switch off. +

+
+ +
+
+ Agent Status + + Click an agent to view details and configuration + +
+ +
+ + + + + Agent + Provider / Type + Drift Score + Iterations (Avg/Max) + Status + Kill Switch + + + + {data.map((agent) => ( + onSelectAgent?.(agent)} + > + + + {agent.name} + + + + + {agent.type} + + + + + {agent.driftScore.toFixed(2)} + + + + + {agent.iterationsAvg} / {agent.iterationsMax} + + + + + + + + + + ))} + +
+ + ); +}; diff --git a/ui/litellm-dashboard/src/components/AgentMonitor/StatsCards.tsx b/ui/litellm-dashboard/src/components/AgentMonitor/StatsCards.tsx new file mode 100644 index 00000000000..600142f6259 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AgentMonitor/StatsCards.tsx @@ -0,0 +1,48 @@ +import { Card, Text, Title } from "@tremor/react"; +import { AlertTriangle, CheckCircle2, Power } from "lucide-react"; +import React from "react"; + +interface StatCardProps { + label: string; + value: string | number; + valueColor?: string; + icon?: React.ReactNode; +} + +const StatCard: React.FC = ({ + label, + value, + valueColor = "text-gray-900", + icon, +}) => ( + +
+ {label} + {icon &&
{icon}
} +
+ {value} + +); + +export const StatsCards: React.FC = () => ( +
+ + } + /> + } + /> + + } + /> +
+); diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index d01fc06bc05..508b2fb06cf 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -175,6 +175,13 @@ const menuGroups: MenuGroup[] = [ icon: , roles: [...all_admin_roles, ...internalUserRoles], }, + { + key: "agent-monitor", + page: "agent-monitor", + label: "Agent Monitor", + icon: , + roles: [...all_admin_roles, ...internalUserRoles], + }, ], }, { diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 6098e75e9a5..49fa9cade71 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2547,6 +2547,7 @@ interface UiSpendLogsParams { key_alias?: string; error_code?: string; error_message?: string; + agent_id?: string; sort_by?: string; sort_order?: "asc" | "desc"; min_spend?: number; diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index a910373d66d..eab1f812460 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -18,6 +18,7 @@ export const pageDescriptions: Record = { new_usage: "View usage analytics and metrics", logs: "Access request and response logs", "guardrails-monitor": "Monitor guardrail performance and view logs", + "agent-monitor": "Monitor agent performance, drift, and behavioral health", users: "Manage internal user accounts and permissions", teams: "Create and manage teams for access control", organizations: "Manage organizations and their members",