From ed913c3a3dfa6c596aa2ad5023a34e1efa517aba Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Sat, 28 Feb 2026 11:57:51 -0800 Subject: [PATCH] feat(perf): add performance debugging dashboard New /v1/performance/summary endpoint + UI dashboard for diagnosing proxy overhead. Backend: - Rolling latency ring buffer (last 100 requests) tracking overhead, llm_api, total ms - Per-model breakdown sorted by overhead - Issue detection: debug logging active, under-provisioned workers, high overhead %, high p95, HTTP pool saturation - Overhead histogram bucketed into 8 ranges - Workers/CPU, DB pool, Redis pool, HTTP pool stats UI: - Performance page at /?page=performance-dashboard (admin only) - Issues Detected table with proposed fixes and collapsible fix snippets - Overhead over time line chart with summary stats (total requests, % under 50ms, p95) - Worker provisioning card with ratio bar - Connections card (DB + Redis pool status) - In-flight asyncio tasks chart - HTTP pool utilization chart - Per-model overhead table --- litellm/constants.py | 3 + litellm/proxy/common_request_processing.py | 6 + .../proxy/performance_endpoints/__init__.py | 0 .../proxy/performance_endpoints/endpoints.py | 324 ++++++++++++ .../performance_endpoints/latency_tracker.py | 186 +++++++ litellm/proxy/proxy_server.py | 2 + .../app/(dashboard)/components/Sidebar2.tsx | 9 + .../usePerformanceSummary.ts | 77 +++ .../performance-dashboard/page.tsx | 17 + ui/litellm-dashboard/src/app/page.tsx | 3 + .../PerformanceDashboardView.tsx | 498 ++++++++++++++++++ .../src/components/leftnav.tsx | 7 + .../src/components/networking.tsx | 26 + 13 files changed, 1158 insertions(+) create mode 100644 litellm/proxy/performance_endpoints/__init__.py create mode 100644 litellm/proxy/performance_endpoints/endpoints.py create mode 100644 litellm/proxy/performance_endpoints/latency_tracker.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/performanceSummary/usePerformanceSummary.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/performance-dashboard/page.tsx create mode 100644 ui/litellm-dashboard/src/components/PerformanceDashboard/PerformanceDashboardView.tsx diff --git a/litellm/constants.py b/litellm/constants.py index 3d2cebf2224..b7b2421caed 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1259,6 +1259,9 @@ RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY = "LiteLLM Virtual Key user_api_key_has # Format: "gen0,gen1,gen2" e.g., "1000,50,50" PYTHON_GC_THRESHOLD = os.getenv("PYTHON_GC_THRESHOLD") +# Performance tracker ring buffer size (number of recent requests to keep latency stats for) +PERF_TRACKER_RING_BUFFER_SIZE = 100 + # pass through route constansts BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES = [ "agents/", diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 1269f58213a..ae54b2e3c99 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -41,6 +41,7 @@ from litellm.proxy.common_utils.callback_utils import ( get_logging_caching_headers, get_remaining_tokens_and_requests_from_request_data, ) +from litellm.proxy.performance_endpoints.latency_tracker import record_request_timing from litellm.proxy.route_llm_request import route_request from litellm.proxy.utils import ProxyLogging from litellm.router import Router @@ -496,6 +497,11 @@ class ProxyBaseLLMRequestProcessing: if logging_caching_headers: headers.update(logging_caching_headers) + record_request_timing( + hidden_params, + model=(request_data or {}).get("model") or None, + ) + try: return { key: str(value) diff --git a/litellm/proxy/performance_endpoints/__init__.py b/litellm/proxy/performance_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/proxy/performance_endpoints/endpoints.py b/litellm/proxy/performance_endpoints/endpoints.py new file mode 100644 index 00000000000..cc92ee2c56f --- /dev/null +++ b/litellm/proxy/performance_endpoints/endpoints.py @@ -0,0 +1,324 @@ +import asyncio +import logging +import multiprocessing +import os +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, Depends + +from litellm._logging import verbose_proxy_logger +from litellm.constants import ( + AIOHTTP_CONNECTOR_LIMIT, + LITELLM_DETAILED_TIMING, +) +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.performance_endpoints.latency_tracker import ( + latency_tracker, + per_model_tracker, +) + +router = APIRouter() + + +@router.get( + "/v1/performance/summary", + tags=["performance"], + dependencies=[Depends(user_api_key_auth)], +) +async def get_performance_summary() -> Dict[str, Any]: + """ + Returns a performance snapshot for diagnosing proxy overhead. + + Sections: + - debug_flags: is DEBUG logging or detailed timing enabled + - workers: cpu_count, configured num_workers, cpu_percent + - connection_pools: in-flight requests, DB pool, Redis pool, HTTP pool + - latency: rolling avg/p50/p95 from the last 100 requests + - per_model: per-model overhead/llm_api/total stats + - issues: ranked list of detected problems with proposed fixes + """ + # --- Debug flags --- + from litellm._logging import verbose_proxy_logger as _proxy_logger + from litellm.proxy.proxy_server import general_settings, prisma_client + + is_detailed_debug = _proxy_logger.isEnabledFor(logging.DEBUG) + log_level = logging.getLevelName(_proxy_logger.getEffectiveLevel()) + detailed_timing_enabled = LITELLM_DETAILED_TIMING + + # --- Workers / CPU --- + cpu_count = multiprocessing.cpu_count() + num_workers = general_settings.get("num_workers", 1) + cpu_percent = _get_cpu_percent() + + # --- Connection pools --- + in_flight_requests = _get_in_flight_requests() + db_pool_info = _get_db_pool_info(general_settings, prisma_client) + redis_pool_info = _get_redis_pool_info() + http_pool_info = _get_http_pool_info() + + # --- Latency stats --- + latency_stats = latency_tracker.stats() + overhead_pct = _compute_overhead_pct(latency_stats) + + # --- Per-model stats --- + per_model_stats = per_model_tracker.stats() + + # --- Issues --- + summary = { + "debug_flags": { + "is_detailed_debug": is_detailed_debug, + "log_level": log_level, + "detailed_timing_enabled": detailed_timing_enabled, + }, + "workers": { + "cpu_count": cpu_count, + "num_workers": num_workers, + "cpu_percent": cpu_percent, + }, + "connection_pools": { + "in_flight_requests": in_flight_requests, + "db": db_pool_info, + "redis": redis_pool_info, + "http": http_pool_info, + }, + "latency": { + **latency_stats, + "overhead_pct_of_total": overhead_pct, + }, + "per_model": per_model_stats, + } + summary["issues"] = _detect_issues(summary) + return summary + + +# --------------------------------------------------------------------------- +# Issue detection +# --------------------------------------------------------------------------- + +_SEVERITY_ORDER = {"critical": 0, "warning": 1, "info": 2} + + +def _detect_issues(summary: Dict[str, Any]) -> List[Dict[str, Any]]: + issues: List[Dict[str, Any]] = [] + + debug_flags = summary["debug_flags"] + workers = summary["workers"] + pools = summary["connection_pools"] + latency = summary["latency"] + + # 1. Debug logging active + if debug_flags["is_detailed_debug"]: + issues.append( + { + "severity": "warning", + "title": "Debug logging is active", + "description": ( + "LITELLM_LOG=DEBUG adds measurable overhead to every request " + "and writes verbose output. Disable in production." + ), + "fix": "Set LITELLM_LOG=WARNING (or unset the variable)", + "fix_snippet": "export LITELLM_LOG=WARNING", + } + ) + + # 2. Under-provisioned workers + cpu_count = workers["cpu_count"] + num_workers = workers["num_workers"] + if num_workers < cpu_count: + recommended = 2 * cpu_count + 1 + issues.append( + { + "severity": "warning", + "title": f"Under-provisioned: {num_workers} worker{'s' if num_workers != 1 else ''} for {cpu_count} CPU cores", + "description": ( + f"You have {cpu_count} CPU cores but only {num_workers} " + f"uvicorn worker{'s' if num_workers != 1 else ''}. " + "Additional workers allow more requests to be handled in parallel." + ), + "fix": f"Set num_workers: {recommended} in your config (2× CPU + 1)", + "fix_snippet": f"litellm --num_workers {recommended} --config config.yaml", + } + ) + + # 3. High overhead percentage + overhead_pct = latency.get("overhead_pct_of_total") + overhead = latency.get("overhead") + if overhead_pct is not None and overhead_pct > 20 and overhead: + issues.append( + { + "severity": "warning", + "title": f"High LiteLLM overhead: {overhead_pct}% of total request time", + "description": ( + f"LiteLLM is adding {overhead['avg_ms']}ms avg overhead " + f"(p95: {overhead['p95_ms']}ms). Normal is <5%. " + "Common causes: debug logging, DB connection pool exhaustion, or too few workers." + ), + "fix": "Check the other issues on this page — high overhead is usually a symptom", + "fix_snippet": None, + } + ) + + # 4. High p95 overhead (even if avg looks okay) + if overhead and overhead.get("p95_ms", 0) > 200: + issues.append( + { + "severity": "warning", + "title": f"High p95 overhead: {overhead['p95_ms']}ms", + "description": ( + "p95 overhead is elevated even if the average looks acceptable. " + "This means 1 in 20 requests experiences significant proxy-added latency. " + "Likely cause: occasional DB pool queuing or GC pauses." + ), + "fix": "Consider increasing database_connection_pool_limit and num_workers", + "fix_snippet": None, + } + ) + + # 6. HTTP pool near saturation + http = pools.get("http", {}) + http_pct = http.get("aiohttp_pct") + if http_pct is not None and http_pct > 80: + issues.append( + { + "severity": "critical", + "title": f"HTTP connection pool near capacity: {http_pct}% used", + "description": ( + f"aiohttp connector is at {http_pct}% utilization " + f"({http.get('aiohttp_active')} / {http.get('aiohttp_limit')} connections). " + "New outbound requests will queue until connections free up." + ), + "fix": "Increase AIOHTTP_CONNECTOR_LIMIT environment variable", + "fix_snippet": f"export AIOHTTP_CONNECTOR_LIMIT={http.get('aiohttp_limit', 300) * 2}", + } + ) + + # Sort: critical first, then warning, then info + issues.sort(key=lambda i: _SEVERITY_ORDER.get(i["severity"], 99)) + return issues + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _get_cpu_percent() -> Optional[float]: + try: + import psutil + + return psutil.cpu_percent(interval=None) + except Exception: + return None + + +def _get_in_flight_requests() -> Optional[int]: + try: + active = sum(1 for t in asyncio.all_tasks() if not t.done()) + return active + except Exception: + return None + + +def _get_db_pool_info( + general_settings: dict, prisma_client: Any +) -> Dict[str, Any]: + pool_limit = general_settings.get("database_connection_pool_limit", 10) + pool_timeout = general_settings.get("database_connection_pool_timeout", 60) + connected = prisma_client is not None + return { + "connected": connected, + "pool_limit": pool_limit, + "pool_timeout_seconds": pool_timeout, + } + + +def _get_redis_pool_info() -> Dict[str, Any]: + try: + from litellm.proxy.proxy_server import redis_usage_cache + + if redis_usage_cache is None: + return {"enabled": False} + result: Dict[str, Any] = {"enabled": True} + try: + if ( + hasattr(redis_usage_cache, "redis_client") + and redis_usage_cache.redis_client + ): + pool = getattr( + redis_usage_cache.redis_client, "connection_pool", None + ) + if pool is not None: + result["max_connections"] = getattr( + pool, "max_connections", None + ) + except Exception: + pass + return result + except Exception: + return {"enabled": False} + + +def _get_http_pool_info() -> Dict[str, Any]: + """ + Returns aiohttp connector pool stats (configured limit + active connections). + Active connections are summed across all cached AsyncHTTPHandler clients. + """ + result: Dict[str, Any] = { + "aiohttp_limit": AIOHTTP_CONNECTOR_LIMIT, + "aiohttp_active": None, + "aiohttp_pct": None, + } + try: + import litellm + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + cache = getattr(litellm, "in_memory_llm_clients_cache", None) + if cache is None: + return result + + # LLMClientCache extends InMemoryCache directly — cache_dict is on cache itself + items = getattr(cache, "cache_dict", {}) + aiohttp_active = 0 + + for _key, client in items.items(): + if not isinstance(client, AsyncHTTPHandler): + continue + httpx_client = getattr(client, "client", None) + if httpx_client is None: + continue + transport = getattr(httpx_client, "_transport", None) + if transport is None: + continue + # LiteLLMAiohttpTransport stores the aiohttp ClientSession as .client + session = getattr(transport, "client", None) + if session is None: + continue + connector = getattr(session, "connector", None) + if connector is None: + continue + acquired = getattr(connector, "_acquired", None) + if acquired is not None: + aiohttp_active += len(acquired) + + result["aiohttp_active"] = aiohttp_active + if AIOHTTP_CONNECTOR_LIMIT > 0: + result["aiohttp_pct"] = round( + aiohttp_active / AIOHTTP_CONNECTOR_LIMIT * 100, 1 + ) + except Exception as e: + verbose_proxy_logger.debug(f"Error getting HTTP pool info: {e}") + + return result + + +def _compute_overhead_pct(latency_stats: dict) -> Optional[float]: + overhead = latency_stats.get("overhead") + total = latency_stats.get("total") + if not overhead or not total: + return None + avg_overhead = overhead.get("avg_ms") + avg_total = total.get("avg_ms") + if avg_overhead and avg_total and avg_total > 0: + return round((avg_overhead / avg_total) * 100, 1) + return None diff --git a/litellm/proxy/performance_endpoints/latency_tracker.py b/litellm/proxy/performance_endpoints/latency_tracker.py new file mode 100644 index 00000000000..c57a88edc27 --- /dev/null +++ b/litellm/proxy/performance_endpoints/latency_tracker.py @@ -0,0 +1,186 @@ +import statistics +import threading +from collections import deque +from typing import Dict, List, Optional + +from litellm.constants import LITELLM_DETAILED_TIMING, PERF_TRACKER_RING_BUFFER_SIZE + +_HISTOGRAM_BUCKETS = [ + (0, 5, "0-5"), + (5, 10, "5-10"), + (10, 25, "10-25"), + (25, 50, "25-50"), + (50, 100, "50-100"), + (100, 200, "100-200"), + (200, 500, "200-500"), + (500, float("inf"), "500+"), +] + + +def _compute_histogram(values: List[float]) -> List[Dict]: + """Returns bucket counts for an overhead latency histogram (ms).""" + counts = {label: 0 for _, _, label in _HISTOGRAM_BUCKETS} + for v in values: + for lo, hi, label in _HISTOGRAM_BUCKETS: + if lo <= v < hi: + counts[label] += 1 + break + return [{"bucket": label, "count": counts[label]} for _, _, label in _HISTOGRAM_BUCKETS] + + +class _LatencyRingBuffer: + """Thread-safe ring buffer storing timing samples from recent requests.""" + + def __init__(self, maxlen: int = PERF_TRACKER_RING_BUFFER_SIZE) -> None: + self._lock = threading.Lock() + self._overhead_ms: deque = deque(maxlen=maxlen) + self._llm_api_ms: deque = deque(maxlen=maxlen) + self._pre_processing_ms: deque = deque(maxlen=maxlen) + self._post_processing_ms: deque = deque(maxlen=maxlen) + self._total_ms: deque = deque(maxlen=maxlen) + + def record( + self, + overhead_ms: Optional[float], + llm_api_ms: Optional[float], + pre_processing_ms: Optional[float], + post_processing_ms: Optional[float], + total_ms: Optional[float], + ) -> None: + with self._lock: + if overhead_ms is not None: + self._overhead_ms.append(overhead_ms) + if llm_api_ms is not None: + self._llm_api_ms.append(llm_api_ms) + if pre_processing_ms is not None: + self._pre_processing_ms.append(pre_processing_ms) + if post_processing_ms is not None: + self._post_processing_ms.append(post_processing_ms) + if total_ms is not None: + self._total_ms.append(total_ms) + + def stats(self) -> Dict: + with self._lock: + overhead_list = list(self._overhead_ms) + return { + "overhead": _compute_stats(overhead_list), + "llm_api": _compute_stats(list(self._llm_api_ms)), + "pre_processing": _compute_stats(list(self._pre_processing_ms)), + "post_processing": _compute_stats(list(self._post_processing_ms)), + "total": _compute_stats(list(self._total_ms)), + "sample_count": len(self._overhead_ms), + "overhead_histogram": _compute_histogram(overhead_list), + } + + +class _PerModelTracker: + """Thread-safe per-model latency tracker using one ring buffer per model.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._models: Dict[str, _LatencyRingBuffer] = {} + + def record( + self, + model: str, + overhead_ms: Optional[float], + llm_api_ms: Optional[float], + total_ms: Optional[float], + ) -> None: + with self._lock: + if model not in self._models: + self._models[model] = _LatencyRingBuffer() + buf = self._models[model] + # record outside the outer lock to minimise contention + buf.record( + overhead_ms=overhead_ms, + llm_api_ms=llm_api_ms, + pre_processing_ms=None, + post_processing_ms=None, + total_ms=total_ms, + ) + + def stats(self) -> List[Dict]: + with self._lock: + snapshot = dict(self._models) + + rows = [] + for model, buf in snapshot.items(): + s = buf.stats() + if s["sample_count"] == 0: + continue + rows.append( + { + "model": model, + "overhead": s["overhead"], + "llm_api": s["llm_api"], + "total": s["total"], + "sample_count": s["sample_count"], + } + ) + # sort by overhead avg descending so worst offenders appear first + rows.sort( + key=lambda r: (r["overhead"] or {}).get("avg_ms", 0), + reverse=True, + ) + return rows + + +def _compute_stats(values: List[float]) -> Optional[Dict]: + if not values: + return None + sorted_vals = sorted(values) + n = len(sorted_vals) + return { + "avg_ms": round(statistics.mean(values), 1), + "p50_ms": round(sorted_vals[n // 2], 1), + "p95_ms": round(sorted_vals[min(int(n * 0.95), n - 1)], 1), + } + + +def _to_float(v: object) -> Optional[float]: + if v is None: + return None + try: + return float(v) # type: ignore[arg-type] + except (ValueError, TypeError): + return None + + +# Module-level singletons — one per worker process +latency_tracker = _LatencyRingBuffer() +per_model_tracker = _PerModelTracker() + + +def record_request_timing(hidden_params: dict, model: Optional[str] = None) -> None: + """ + Call once per completed request with the response's hidden_params. + Extracts timing fields and appends to the ring buffers (global + per-model). + """ + total_ms = _to_float(hidden_params.get("_response_ms")) + overhead_ms = _to_float(hidden_params.get("litellm_overhead_time_ms")) + + # Derive llm_api_ms: prefer explicit timing, fall back to total - overhead + llm_api_ms = _to_float(hidden_params.get("timing_llm_api_ms")) + if llm_api_ms is None and total_ms is not None and overhead_ms is not None: + llm_api_ms = max(0.0, total_ms - overhead_ms) + + latency_tracker.record( + overhead_ms=overhead_ms, + llm_api_ms=llm_api_ms, + pre_processing_ms=_to_float(hidden_params.get("timing_pre_processing_ms")) + if LITELLM_DETAILED_TIMING + else None, + post_processing_ms=_to_float(hidden_params.get("timing_post_processing_ms")) + if LITELLM_DETAILED_TIMING + else None, + total_ms=total_ms, + ) + + if model: + per_model_tracker.record( + model=model, + overhead_ms=overhead_ms, + llm_api_ms=llm_api_ms, + total_ms=total_ms, + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index be76c2ac5fb..a839d3af2a6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -448,6 +448,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( router as pass_through_router, ) +from litellm.proxy.performance_endpoints.endpoints import router as performance_router from litellm.proxy.policy_engine.policy_endpoints import router as policy_crud_router from litellm.proxy.policy_engine.policy_resolve_endpoints import ( router as policy_resolve_router, @@ -13027,3 +13028,4 @@ async def dynamic_mcp_route(mcp_server_name: str, request: Request): app.mount(path=BASE_MCP_ROUTE, app=mcp_app) app.include_router(mcp_rest_endpoints_router) app.include_router(mcp_discoverable_endpoints_router) +app.include_router(performance_router) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx b/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx index a74d3c108d6..4fead1dbbbc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/components/Sidebar2.tsx @@ -101,6 +101,8 @@ const routeFor = (slug: string): string => { return "model-hub"; case "logs": return "logs"; + case "performance-dashboard": + return "performance-dashboard"; case "guardrails": return "guardrails"; case "policies": @@ -198,6 +200,13 @@ const menuItems: MenuItemCfg[] = [ icon: , }, { key: "15", page: "logs", label: "Logs", icon: }, + { + key: "30", + page: "performance-dashboard", + label: "Performance", + icon: , + roles: all_admin_roles, + }, { key: "11", page: "guardrails", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/performanceSummary/usePerformanceSummary.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/performanceSummary/usePerformanceSummary.ts new file mode 100644 index 00000000000..3d848b19e1a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/performanceSummary/usePerformanceSummary.ts @@ -0,0 +1,77 @@ +import { useQuery, UseQueryResult } from "@tanstack/react-query"; +import { getPerformanceSummaryCall } from "@/components/networking"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +export interface LatencyStats { + avg_ms: number; + p50_ms: number; + p95_ms: number; +} + +export interface PerModelStats { + model: string; + overhead: LatencyStats | null; + llm_api: LatencyStats | null; + total: LatencyStats | null; + sample_count: number; +} + +export interface PerformanceIssue { + severity: "critical" | "warning" | "info"; + title: string; + description: string; + fix: string; + fix_snippet: string | null; +} + +export interface PerformanceSummaryResponse { + debug_flags: { + is_detailed_debug: boolean; + log_level: string; + detailed_timing_enabled: boolean; + }; + workers: { + cpu_count: number; + num_workers: number; + cpu_percent: number | null; + }; + connection_pools: { + in_flight_requests: number | null; + db: { + connected: boolean; + pool_limit: number; + pool_timeout_seconds: number; + }; + redis: { + enabled: boolean; + max_connections?: number | null; + }; + http: { + aiohttp_limit: number; + aiohttp_active: number | null; + aiohttp_pct: number | null; + }; + }; + latency: { + overhead: LatencyStats | null; + llm_api: LatencyStats | null; + pre_processing: LatencyStats | null; + post_processing: LatencyStats | null; + total: LatencyStats | null; + overhead_pct_of_total: number | null; + sample_count: number; + overhead_histogram: { bucket: string; count: number }[]; + }; + per_model: PerModelStats[]; + issues: PerformanceIssue[]; +} + +export const usePerformanceSummary = (): UseQueryResult => { + const { accessToken } = useAuthorized(); + return useQuery({ + queryKey: ["performanceSummary"], + queryFn: async () => getPerformanceSummaryCall(accessToken!), + enabled: Boolean(accessToken), + refetchInterval: 10_000, + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/performance-dashboard/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/performance-dashboard/page.tsx new file mode 100644 index 00000000000..e31124e3f31 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/performance-dashboard/page.tsx @@ -0,0 +1,17 @@ +"use client"; + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import PerformanceDashboardView from "@/components/PerformanceDashboard/PerformanceDashboardView"; + +const PerformanceDashboardPage = () => { + useAuthorized(); + const queryClient = new QueryClient(); + return ( + + + + ); +}; + +export default PerformanceDashboardPage; diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 0b2f467e8f8..e2dee36ca3b 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -14,6 +14,7 @@ import LoadingScreen from "@/components/common_components/LoadingScreen"; import { CostTrackingSettings } from "@/components/CostTrackingSettings"; import GeneralSettings from "@/components/general_settings"; import GuardrailsMonitorView from "@/components/GuardrailsMonitor/GuardrailsMonitorView"; +import PerformanceDashboardView from "@/components/PerformanceDashboard/PerformanceDashboardView"; import GuardrailsPanel from "@/components/guardrails"; import PoliciesPanel from "@/components/policies"; import { Team } from "@/components/key_team_helpers/key_list"; @@ -556,6 +557,8 @@ function CreateKeyPageContent() { ) : page == "guardrails-monitor" ? ( + ) : page == "performance-dashboard" ? ( + ) : page == "new_usage" ? ( = { + critical: "bg-red-500", + warning: "bg-amber-400", + info: "bg-blue-400", +}; + +export default function PerformanceDashboardView() { + const { data, isLoading, error, dataUpdatedAt } = usePerformanceSummary(); + + // Client-side ring buffer for time-series charts + const overheadHistoryRef = useRef([]); + const [overheadHistory, setOverheadHistory] = useState([]); + + const inflightHistoryRef = useRef([]); + const [inflightHistory, setInflightHistory] = useState([]); + + const httpHistoryRef = useRef([]); + const [httpHistory, setHttpHistory] = useState([]); + + // Live "X seconds ago" counter + const [secondsAgo, setSecondsAgo] = useState(0); + useEffect(() => { + if (!dataUpdatedAt) return; + setSecondsAgo(0); + const interval = setInterval(() => { + setSecondsAgo(Math.floor((Date.now() - dataUpdatedAt) / 1000)); + }, 1000); + return () => clearInterval(interval); + }, [dataUpdatedAt]); + + useEffect(() => { + if (!data) return; + const now = formatTime(new Date()); + + const newOH = [...overheadHistoryRef.current, { + time: now, + "Overhead avg": data.latency.overhead?.avg_ms ?? undefined, + }].slice(-MAX_HISTORY); + overheadHistoryRef.current = newOH; + setOverheadHistory([...newOH]); + + const newIF = [...inflightHistoryRef.current, { + time: now, + "In-flight": data.connection_pools.in_flight_requests ?? undefined, + }].slice(-MAX_HISTORY); + inflightHistoryRef.current = newIF; + setInflightHistory([...newIF]); + + const newHTTP = [...httpHistoryRef.current, { + time: now, + "HTTP pool %": data.connection_pools.http.aiohttp_pct ?? undefined, + }].slice(-MAX_HISTORY); + httpHistoryRef.current = newHTTP; + setHttpHistory([...newHTTP]); + }, [dataUpdatedAt]); + + if (isLoading) { + return ( +
+ +
+ ); + } + + if (error || !data) { + return ( +
+ +
+ ); + } + + const { debug_flags, workers, connection_pools, latency, per_model, issues } = data; + const overheadHigh = latency.overhead_pct_of_total != null && latency.overhead_pct_of_total > 20; + const workersLow = workers.num_workers < workers.cpu_count; + + // Delta vs ~5 min ago (oldest point in history) + const oldestOverhead = overheadHistory.length > 1 + ? overheadHistory[0]["Overhead avg"] + : null; + const currentOverhead = latency.overhead?.avg_ms ?? null; + const overheadDelta = oldestOverhead != null && currentOverhead != null + ? Math.round((currentOverhead - oldestOverhead) * 10) / 10 + : null; + + return ( +
+ {/* Header */} +
+
+
+ +
+
+

Performance

+

LiteLLM Proxy · Latency Diagnostics

+
+
+ + {dataUpdatedAt + ? secondsAgo === 0 ? "Refreshed just now" : `Refreshed ${secondsAgo}s ago` + : "Waiting for data…"} + +
+ + {/* ── Issues Detected ── */} +
+ +
+

Issues Detected

+ {issues.length === 0 && latency.sample_count > 0 && ( + + All clear + + )} +
+ {issues.length === 0 ? ( +

+ {latency.sample_count === 0 ? "Send traffic to start analysis." : "No issues found — proxy looks healthy."} +

+ ) : ( + + + + + + + + + + {issues.map((issue, i) => ( + + + + + + ))} + +
IssueSuggested Fix
+ + +

{issue.title}

+

{issue.description}

+ 		+ {issue.fix_snippet ? ( + + {issue.fix}} key="1"> + 
+                              {issue.fix_snippet}
+
+ + + ) : ( +

{issue.fix}

+ )} +
+ )} + +
+ + {/* ── Top metric cards ── */} +
+ = 0 ? "+" : ""}${overheadDelta}ms vs earlier` : ""}` + : "No data yet" + } + valueColor={overheadHigh ? "text-orange-500" : "text-gray-900"} + /> + + + +
+ + {/* ── Main 2-column layout ── */} +
+ {/* Left: Overhead Over Time */} +
+ +

Overhead Over Time

+

LiteLLM-added latency · last 10 min · 10s resolution

+ + {/* Plain-English summary */} + {latency.sample_count > 0 && latency.overhead_histogram && (() => { + const hist = latency.overhead_histogram; + const total = hist.reduce((s, b) => s + b.count, 0); + const under50 = hist.filter(b => ["0-5","5-10","10-25","25-50"].includes(b.bucket)).reduce((s,b) => s+b.count, 0); + const pct = total > 0 ? Math.round(under50 / total * 100) : 0; + const worstBucket = [...hist].reverse().find(b => b.count > 0); + return ( +
+
+

{total}

+

total requests

+
+
+

= 90 ? "text-green-600" : pct >= 70 ? "text-amber-500" : "text-red-500"}`}>{pct}%

+

under 50ms overhead

+
+
+

{latency.overhead?.p95_ms ?? "—"}ms

+

p95 overhead

+
+ {worstBucket && worstBucket.bucket !== "0-5" && ( +
+

{worstBucket.count}

+

requests > {worstBucket.bucket.split("-")[0]}ms

+
+ )} +
+ ); + })()} + + {latency.sample_count === 0 ? ( +
+ No requests yet — send traffic through the proxy to see data. +
+ ) : ( + `${v} ms`} + yAxisWidth={52} + showLegend={false} + showAnimation={false} + className="h-40" + /> + )} + +
+ + {/* Right: Worker Provisioning + Connections */} +
+ +
+

WORKER PROVISIONING

+ {workersLow && Under-provisioned} +
+
+
+

CPU Cores

+

{workers.cpu_count}

+
+
+

Workers

+

+ {workers.num_workers} +

+
+
+ {workers.cpu_percent != null && ( +
+

CPU Usage

+

80 ? "text-red-500" : workers.cpu_percent > 60 ? "text-orange-500" : "text-green-600"}`}> + {workers.cpu_percent}% +

+
+ )} +
+ Workers / CPU ratio + {workers.num_workers}/{workers.cpu_count} +
+
+
+
+ {workersLow && ( +

Recommended: {2 * workers.cpu_count + 1} workers (2× CPU + 1)

+ )} + + + +

CONNECTIONS

+
+
+
+

Database

+ + {connection_pools.db.connected ? "● Connected" : "● Disconnected"} + +
+
+

Pool limit

+

{connection_pools.db.pool_limit}

+
+
+
+ Timeout: {connection_pools.db.pool_timeout_seconds}s +
+
+
+
+

Redis

+ + {connection_pools.redis.enabled ? "● Connected" : "● Not configured"} + +
+ {connection_pools.redis.enabled && ( +
+

Max connections

+

{connection_pools.redis.max_connections ?? "∞"}

+
+ )} +
+
+
+ +
+
+ + {/* ── Bottom row: In-Flight + HTTP pool ── */} +
+ +
+

In-Flight Requests

+ + {connection_pools.in_flight_requests ?? "—"} + asyncio tasks + +
+

+ Concurrent asyncio tasks (proxy-wide) · last 10 min +

+ + + + +
+

HTTP Client Pool Utilization

+ + {connection_pools.http.aiohttp_active ?? "—"} + + {" "}/ {connection_pools.http.aiohttp_limit} limit + {connection_pools.http.aiohttp_pct != null && ` · ${connection_pools.http.aiohttp_pct}%`} + + +
+

+ aiohttp active connections ÷ pool limit · amber line = 80% · last 10 min +

+ `${v}%`} + yAxisWidth={44} + showLegend={false} + showAnimation={false} + className="h-40" + referenceLine={{ value: 80, label: "80%", color: "amber" }} + /> + +
+ + {/* ── Per-model breakdown ── */} + {per_model.length > 0 && ( +
+ +

Per-Model Overhead

+

Sorted by overhead avg · last {latency.sample_count} requests

+
+ + + + + + + + + + + + + {per_model.map((row) => { + const overheadPct = row.overhead && row.total + ? Math.round((row.overhead.avg_ms / row.total.avg_ms) * 100) + : null; + return ( + + + + + + + + + ); + })} + +
ModelOverhead avgOverhead p95LLM API avgTotal avgRequests
+ {row.model} + + 20 ? "text-orange-500 font-semibold" : "text-gray-700"}> + {row.overhead ? `${row.overhead.avg_ms}ms` : "—"} + + {overheadPct != null && ( + ({overheadPct}%) + )} + + {row.overhead ? `${row.overhead.p95_ms}ms` : "—"} + + {row.llm_api ? `${row.llm_api.avg_ms}ms` : "—"} + + {row.total ? `${row.total.avg_ms}ms` : "—"} + {row.sample_count}
+
+ +
+ )} + + {/* ── Config summary ── */} +
+

Configuration Summary

+
+
+

Log Level

+ {debug_flags.log_level} +
+
+

Detailed Timing

+ + {debug_flags.detailed_timing_enabled ? "Enabled" : "Disabled"} + +
+
+

Workers / CPU

+ {workers.num_workers} / {workers.cpu_count} +
+
+

Sample Count

+ {latency.sample_count} +
+
+
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index bb0bd54c7a9..1b94380d5be 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -168,6 +168,13 @@ const menuGroups: MenuGroup[] = [ icon: , roles: [...all_admin_roles, ...internalUserRoles], }, + { + key: "performance-dashboard", + page: "performance-dashboard", + label: "Performance", + icon: , + roles: all_admin_roles, + }, ], }, { diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 0df6d813d7c..3a47b5eab80 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -10035,3 +10035,29 @@ export const updateToolPolicy = async ( } return response.json(); }; + + +export const getPerformanceSummaryCall = async (accessToken: string) => { + const url = proxyBaseUrl + ? `${proxyBaseUrl}/v1/performance/summary` + : `/v1/performance/summary`; + try { + const response = await fetch(url, { + method: "GET", + headers: { + [globalLitellmHeaderName]: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + if (!response.ok) { + const errorData = await response.json(); + const errorMessage = deriveErrorMessage(errorData); + handleError(errorMessage); + throw new Error(errorMessage); + } + return await response.json(); + } catch (error) { + console.error("Failed to get performance summary:", error); + throw error; + } +};