From 2ce708490d8cf8eabe9cdb65df95814062dd8e42 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Sat, 28 Feb 2026 12:52:15 -0800 Subject: [PATCH] ui(perf): two-tab layout with Postman-style timing waterfall - Move Test Request into its own tab (Overview | Test Request) - Reduce color - muted gray palette, orange only for high overhead - Add Postman-style waterfall: proportional bars for LiteLLM Processing vs LLM API - Message editor with role selector and multi-message support - Status bar showing HTTP status, wall time, model name - x-litellm headers table shown below response --- .../PerformanceDashboardView.tsx | 781 +++++++++--------- 1 file changed, 390 insertions(+), 391 deletions(-) diff --git a/ui/litellm-dashboard/src/components/PerformanceDashboard/PerformanceDashboardView.tsx b/ui/litellm-dashboard/src/components/PerformanceDashboard/PerformanceDashboardView.tsx index 5ac172d9c94..07f7082b27b 100644 --- a/ui/litellm-dashboard/src/components/PerformanceDashboard/PerformanceDashboardView.tsx +++ b/ui/litellm-dashboard/src/components/PerformanceDashboard/PerformanceDashboardView.tsx @@ -1,17 +1,22 @@ "use client"; -import React, { useRef, useState, useEffect } from "react"; -import { Collapse, Spin, Tag } from "antd"; -import { DashboardOutlined, CheckCircleOutlined } from "@ant-design/icons"; -import { BarChart, LineChart, Card } from "@tremor/react"; -import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; +import React, { useCallback, useRef, useState, useEffect } from "react"; +import { Collapse, Select, Spin, Tabs, Tag } from "antd"; +import { + CheckCircleOutlined, + CloseCircleOutlined, + DashboardOutlined, + PlayCircleOutlined, +} from "@ant-design/icons"; +import { LineChart, Card } from "@tremor/react"; import { usePerformanceSummary, - PerformanceSummaryResponse, - PerformanceIssue, } from "@/app/(dashboard)/hooks/performanceSummary/usePerformanceSummary"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { fetchAvailableModels, ModelGroup } from "@/components/playground/llm_calls/fetch_models"; +import { proxyBaseUrl, getGlobalLitellmHeaderName } from "@/components/networking"; -const MAX_HISTORY = 60; // 60 × 10s = 10 min +const MAX_HISTORY = 60; interface HistoryPoint { time: string; @@ -20,6 +25,15 @@ interface HistoryPoint { "HTTP pool %"?: number; } +interface TestResult { + status: number; + overheadMs: number | null; + wallMs: number; + model: string; + responseText: string; + headers: Record; +} + function formatTime(d: Date): string { return d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" }); } @@ -30,130 +44,141 @@ const SEVERITY_DOT: Record = { info: "bg-blue-400", }; +async function runTestRequest(accessToken: string, model: string, messages: { role: string; content: string }[]): Promise { + const url = proxyBaseUrl ? `${proxyBaseUrl}/chat/completions` : `/chat/completions`; + const authHeader = getGlobalLitellmHeaderName(); + const start = Date.now(); + const resp = await fetch(url, { + method: "POST", + headers: { [authHeader]: `Bearer ${accessToken}`, "Content-Type": "application/json" }, + body: JSON.stringify({ model, messages, max_tokens: 100, stream: false }), + }); + const wallMs = Date.now() - start; + const responseText = await resp.text(); + const captured: Record = {}; + resp.headers.forEach((v, k) => { if (k.startsWith("x-litellm")) captured[k] = v; }); + const overheadRaw = resp.headers.get("x-litellm-overhead-duration-ms"); + return { + status: resp.status, + overheadMs: overheadRaw ? Math.round(parseFloat(overheadRaw)) : null, + wallMs, + model, + responseText, + headers: captured, + }; +} + +// Postman-style waterfall row +function WaterfallRow({ label, startMs, durationMs, totalMs, color }: { + label: string; + startMs: number; + durationMs: number; + totalMs: number; + color: string; +}) { + const startPct = totalMs > 0 ? (startMs / totalMs) * 100 : 0; + const widthPct = totalMs > 0 ? (durationMs / totalMs) * 100 : 0; + return ( + + {label} + +
+
+
+ + {durationMs}ms + + ); +} + export default function PerformanceDashboardView() { const { data, isLoading, error, dataUpdatedAt } = usePerformanceSummary(); + const { accessToken } = useAuthorized(); - // Client-side ring buffer for time-series charts const overheadHistoryRef = useRef([]); const [overheadHistory, setOverheadHistory] = useState([]); - const inflightHistoryRef = useRef([]); const [inflightHistory, setInflightHistory] = useState([]); - const httpHistoryRef = useRef([]); const [httpHistory, setHttpHistory] = useState([]); - // Live "X seconds ago" counter const [secondsAgo, setSecondsAgo] = useState(0); useEffect(() => { if (!dataUpdatedAt) return; setSecondsAgo(0); - const interval = setInterval(() => { - setSecondsAgo(Math.floor((Date.now() - dataUpdatedAt) / 1000)); - }, 1000); + const interval = setInterval(() => setSecondsAgo(Math.floor((Date.now() - dataUpdatedAt) / 1000)), 1000); return () => clearInterval(interval); }, [dataUpdatedAt]); useEffect(() => { if (!data) return; const now = formatTime(new Date()); - - const newOH = [...overheadHistoryRef.current, { - time: now, - "Overhead avg": data.latency.overhead?.avg_ms ?? undefined, - }].slice(-MAX_HISTORY); - overheadHistoryRef.current = newOH; - setOverheadHistory([...newOH]); - - const newIF = [...inflightHistoryRef.current, { - time: now, - "In-flight": data.connection_pools.in_flight_requests ?? undefined, - }].slice(-MAX_HISTORY); - inflightHistoryRef.current = newIF; - setInflightHistory([...newIF]); - - const newHTTP = [...httpHistoryRef.current, { - time: now, - "HTTP pool %": data.connection_pools.http.aiohttp_pct ?? undefined, - }].slice(-MAX_HISTORY); - httpHistoryRef.current = newHTTP; - setHttpHistory([...newHTTP]); + const newOH = [...overheadHistoryRef.current, { time: now, "Overhead avg": data.latency.overhead?.avg_ms ?? undefined }].slice(-MAX_HISTORY); + overheadHistoryRef.current = newOH; setOverheadHistory([...newOH]); + const newIF = [...inflightHistoryRef.current, { time: now, "In-flight": data.connection_pools.in_flight_requests ?? undefined }].slice(-MAX_HISTORY); + inflightHistoryRef.current = newIF; setInflightHistory([...newIF]); + const newHTTP = [...httpHistoryRef.current, { time: now, "HTTP pool %": data.connection_pools.http.aiohttp_pct ?? undefined }].slice(-MAX_HISTORY); + httpHistoryRef.current = newHTTP; setHttpHistory([...newHTTP]); }, [dataUpdatedAt]); - if (isLoading) { - return ( -
- -
- ); - } + // Test Request state + const [models, setModels] = useState([]); + const [selectedModel, setSelectedModel] = useState(undefined); + const [testMessages, setTestMessages] = useState([{ role: "user", content: "Say hello in one word." }]); + const [isTestLoading, setIsTestLoading] = useState(false); + const [testResult, setTestResult] = useState(null); + const [testError, setTestError] = useState(null); - if (error || !data) { - return ( -
- -
- ); - } + useEffect(() => { + if (!accessToken) return; + fetchAvailableModels(accessToken).then((m) => { + const chat = m.filter((x) => !x.mode || x.mode === "chat"); + const list = chat.length > 0 ? chat : m; + setModels(list); + if (list.length > 0 && !selectedModel) setSelectedModel(list[0].model_group); + }).catch(() => {}); + }, [accessToken]); - const { debug_flags, workers, connection_pools, latency, per_model, issues } = data; + const handleRunTest = useCallback(async () => { + if (!accessToken || !selectedModel) return; + setIsTestLoading(true); setTestError(null); setTestResult(null); + try { + setTestResult(await runTestRequest(accessToken, selectedModel, testMessages)); + } catch (e: any) { + setTestError(e.message ?? "Request failed"); + } finally { + setIsTestLoading(false); + } + }, [accessToken, selectedModel, testMessages]); + + if (isLoading) return
; + if (error || !data) return
Failed to load performance data.
; + + const { debug_flags, workers, connection_pools, latency, issues } = data; const overheadHigh = latency.overhead_pct_of_total != null && latency.overhead_pct_of_total > 20; const workersLow = workers.num_workers < workers.cpu_count; - // Delta vs ~5 min ago (oldest point in history) - const oldestOverhead = overheadHistory.length > 1 - ? overheadHistory[0]["Overhead avg"] - : null; - const currentOverhead = latency.overhead?.avg_ms ?? null; - const overheadDelta = oldestOverhead != null && currentOverhead != null - ? Math.round((currentOverhead - oldestOverhead) * 10) / 10 - : null; - - return ( -
- {/* Header */} -
-
-
- -
-
-

Performance

-

LiteLLM Proxy · Latency Diagnostics

-
-
- - {dataUpdatedAt - ? secondsAgo === 0 ? "Refreshed just now" : `Refreshed ${secondsAgo}s ago` - : "Waiting for data…"} - -
- - {/* ── Issues Detected ── */} + const overviewTab = ( +
+ {/* Issues Detected */}

Issues Detected

{issues.length === 0 && latency.sample_count > 0 && ( - - All clear - + All clear )}
{issues.length === 0 ? ( -

- {latency.sample_count === 0 ? "Send traffic to start analysis." : "No issues found — proxy looks healthy."} -

+

{latency.sample_count === 0 ? "Send traffic to start analysis." : "No issues found — proxy looks healthy."}

) : ( - + @@ -161,20 +186,16 @@ export default function PerformanceDashboardView() { {issues.map((issue, i) => ( - + -
Issue Suggested Fix
- -

{issue.title}

{issue.description}

 + {issue.fix_snippet ? ( {issue.fix}} key="1"> - 
-                              {issue.fix_snippet}
-
+ 
{issue.fix_snippet}
 ) : ( @@ -189,310 +210,288 @@ export default function PerformanceDashboardView() { - {/* ── Top metric cards ── */} -
- = 0 ? "+" : ""}${overheadDelta}ms vs earlier` : ""}` - : "No data yet" - } - valueColor={overheadHigh ? "text-orange-500" : "text-gray-900"} - /> - - - -
- - {/* ── Main 2-column layout ── */} -
- {/* Left: Overhead Over Time */} -
- -

Overhead Over Time

-

LiteLLM-added latency · last 10 min · 10s resolution

- - {/* Plain-English summary */} - {latency.sample_count > 0 && latency.overhead_histogram && (() => { - const hist = latency.overhead_histogram; - const total = hist.reduce((s, b) => s + b.count, 0); - const under50 = hist.filter(b => ["0-5","5-10","10-25","25-50"].includes(b.bucket)).reduce((s,b) => s+b.count, 0); - const pct = total > 0 ? Math.round(under50 / total * 100) : 0; - const worstBucket = [...hist].reverse().find(b => b.count > 0); - return ( -
-
-

{total}

-

total requests

-
-
-

= 90 ? "text-green-600" : pct >= 70 ? "text-amber-500" : "text-red-500"}`}>{pct}%

-

under 50ms overhead

-
-
-

{latency.overhead?.p95_ms ?? "—"}ms

-

p95 overhead

-
- {worstBucket && worstBucket.bucket !== "0-5" && ( -
-

{worstBucket.count}

-

requests > {worstBucket.bucket.split("-")[0]}ms

-
- )} -
- ); - })()} - - {latency.sample_count === 0 ? ( -
- No requests yet — send traffic through the proxy to see data. -
- ) : ( - `${v} ms`} - yAxisWidth={52} - showLegend={false} - showAnimation={false} - className="h-40" - /> - )} - -
- - {/* Right: Worker Provisioning + Connections */} -
- -
-

WORKER PROVISIONING

- {workersLow && Under-provisioned} -
-
-
-

CPU Cores

-

{workers.cpu_count}

-
-
-

Workers

-

- {workers.num_workers} -

-
-
- {workers.cpu_percent != null && ( -
-

CPU Usage

-

80 ? "text-red-500" : workers.cpu_percent > 60 ? "text-orange-500" : "text-green-600"}`}> - {workers.cpu_percent}% -

-
- )} -
- Workers / CPU ratio - {workers.num_workers}/{workers.cpu_count} -
-
-
-
- {workersLow && ( -

Recommended: {2 * workers.cpu_count + 1} workers (2× CPU + 1)

- )} - - - -

CONNECTIONS

-
-
-
-

Database

- - {connection_pools.db.connected ? "● Connected" : "● Disconnected"} - -
-
-

Pool limit

-

{connection_pools.db.pool_limit}

-
-
-
- Timeout: {connection_pools.db.pool_timeout_seconds}s -
-
-
-
-

Redis

- - {connection_pools.redis.enabled ? "● Connected" : "● Not configured"} - -
- {connection_pools.redis.enabled && ( -
-

Max connections

-

{connection_pools.redis.max_connections ?? "∞"}

-
- )} -
-
-
- -
-
- - {/* ── Bottom row: In-Flight + HTTP pool ── */} + {/* Current Overhead | Worker Provisioning */}
-
-

In-Flight Requests

- - {connection_pools.in_flight_requests ?? "—"} - asyncio tasks - -
-

- Concurrent asyncio tasks (proxy-wide) · last 10 min -

- +

Current Overhead

+ {latency.sample_count === 0 ? ( +

No data — send traffic through the proxy.

+ ) : ( + <> +
+

Avg

{latency.overhead ? `${latency.overhead.avg_ms}ms` : "—"}

+

p50

{latency.overhead ? `${latency.overhead.p50_ms}ms` : "—"}

+

p95

{latency.overhead ? `${latency.overhead.p95_ms}ms` : "—"}

+

% of total

{latency.overhead_pct_of_total != null ? `${latency.overhead_pct_of_total}%` : "—"}

+
+

Last {latency.sample_count} requests · LLM API avg: {latency.llm_api ? `${latency.llm_api.avg_ms}ms` : "—"} · Total avg: {latency.total ? `${latency.total.avg_ms}ms` : "—"}

+ `${v}ms`} yAxisWidth={48} showLegend={false} showAnimation={false} className="h-28" /> + + )} -
-

HTTP Client Pool Utilization

- - {connection_pools.http.aiohttp_active ?? "—"} - - {" "}/ {connection_pools.http.aiohttp_limit} limit - {connection_pools.http.aiohttp_pct != null && ` · ${connection_pools.http.aiohttp_pct}%`} - - +
+

Worker Provisioning

+ {workersLow && Under-provisioned}
-

- aiohttp active connections ÷ pool limit · amber line = 80% · last 10 min -

- `${v}%`} - yAxisWidth={44} - showLegend={false} - showAnimation={false} - className="h-40" - referenceLine={{ value: 80, label: "80%", color: "amber" }} - /> +
+

CPU Cores

{workers.cpu_count}

+

Workers

{workers.num_workers}

+
+

CPU Usage

+

80 ? "text-red-500" : workers.cpu_percent > 60 ? "text-orange-500" : "text-green-600"}`}> + {workers.cpu_percent != null ? `${workers.cpu_percent}%` : "—"} +

+
+
+
Workers / CPU {workers.num_workers}/{workers.cpu_count}
+
+
+
+ {workersLow &&

Recommended: {2 * workers.cpu_count + 1} workers (2× CPU + 1)

}
- {/* ── Per-model breakdown ── */} - {per_model.length > 0 && ( -
- -

Per-Model Overhead

-

Sorted by overhead avg · last {latency.sample_count} requests

-
- - - - - - - - - - - - - {per_model.map((row) => { - const overheadPct = row.overhead && row.total - ? Math.round((row.overhead.avg_ms / row.total.avg_ms) * 100) - : null; - return ( - - - - - - - - - ); - })} - -
ModelOverhead avgOverhead p95LLM API avgTotal avgRequests
- {row.model} - - 20 ? "text-orange-500 font-semibold" : "text-gray-700"}> - {row.overhead ? `${row.overhead.avg_ms}ms` : "—"} - - {overheadPct != null && ( - ({overheadPct}%) - )} - - {row.overhead ? `${row.overhead.p95_ms}ms` : "—"} - - {row.llm_api ? `${row.llm_api.avg_ms}ms` : "—"} - - {row.total ? `${row.total.avg_ms}ms` : "—"} - {row.sample_count}
-
- -
- )} + {/* Proxy Settings */} +
+ +

Proxy Settings

+ + + + + + + + + + + + + + + + + + + + + + + + + +
SettingCurrent ValueOptimized for Performance
Debug Mode{debug_flags.log_level}{!debug_flags.is_detailed_debug ? Yes : No — disable DEBUG logging in production}
Detailed Timing Headers{debug_flags.detailed_timing_enabled ? "Enabled" : "Disabled"}{!debug_flags.detailed_timing_enabled ? Yes : Minor overhead — safe to keep for debugging}
Worker Count{workers.num_workers}/ {workers.cpu_count} CPU cores{!workersLow ? Yes : No — recommend {2 * workers.cpu_count + 1} workers (2× CPU + 1)}
+ +
- {/* ── Config summary ── */} -
-

Configuration Summary

-
-
-

Log Level

- {debug_flags.log_level} + {/* Active Async Tasks | HTTP Pool */} +
+ +
+

Active Async Tasks

+ {connection_pools.in_flight_requests ?? "—"} tasks
-
-

Detailed Timing

- - {debug_flags.detailed_timing_enabled ? "Enabled" : "Disabled"} - +

Total asyncio tasks running on this worker · last 10 min

+ + + +
+

HTTP Client Pool

+ {connection_pools.http.aiohttp_active ?? "—"} / {connection_pools.http.aiohttp_limit}{connection_pools.http.aiohttp_pct != null && ` · ${connection_pools.http.aiohttp_pct}%`}
-
-

Workers / CPU

- {workers.num_workers} / {workers.cpu_count} -
-
-

Sample Count

- {latency.sample_count} -
-
+

aiohttp active / pool limit · last 10 min

+ `${v}%`} yAxisWidth={44} showLegend={false} showAnimation={false} className="h-36" referenceLine={{ value: 80, label: "80%", color: "amber" }} /> +
); + + const llmApiMs = testResult ? Math.max(0, testResult.wallMs - (testResult.overheadMs ?? 0)) : 0; + const overheadPctTest = testResult && testResult.overheadMs != null + ? Math.round((testResult.overheadMs / testResult.wallMs) * 100) + : null; + + let parsedContent = ""; + if (testResult) { + try { + const parsed = JSON.parse(testResult.responseText); + parsedContent = parsed?.choices?.[0]?.message?.content ?? testResult.responseText; + } catch { + parsedContent = testResult.responseText; + } + } + + const testTab = ( +
+ + {/* Controls */} +
+
+

Model

+ setTestMessages((prev) => prev.map((m, j) => j === i ? { ...m, role: e.target.value } : m))} + className="text-xs border border-gray-200 rounded px-2 py-1.5 text-gray-600 bg-white w-24 flex-shrink-0" + > + + + + +