From 10e88a939db9750cb482bfdbe996a5d2be729056 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 21 Jun 2024 13:08:54 -0700 Subject: [PATCH 1/8] backend - new endpoint to show cache hit stats --- .../analytics_endpoints.py | 103 ++++++++++++++++++ litellm/proxy/proxy_server.py | 4 + 2 files changed, 107 insertions(+) create mode 100644 litellm/proxy/analytics_endpoints/analytics_endpoints.py diff --git a/litellm/proxy/analytics_endpoints/analytics_endpoints.py b/litellm/proxy/analytics_endpoints/analytics_endpoints.py new file mode 100644 index 00000000000..ebb7d43acfd --- /dev/null +++ b/litellm/proxy/analytics_endpoints/analytics_endpoints.py @@ -0,0 +1,103 @@ +#### Analytics Endpoints ##### +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +import fastapi +from fastapi import APIRouter, Depends, Header, HTTPException, Request, status + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import * +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +router = APIRouter() + + +@router.get( + "/global/activity/cache_hits", + tags=["Budget & Spend Tracking"], + dependencies=[Depends(user_api_key_auth)], + responses={ + 200: {"model": List[LiteLLM_SpendLogs]}, + }, + include_in_schema=False, +) +async def get_global_activity( + start_date: Optional[str] = fastapi.Query( + default=None, + description="Time from which to start viewing spend", + ), + end_date: Optional[str] = fastapi.Query( + default=None, + description="Time till which to view spend", + ), +): + """ + Get number of cache hits, vs misses + + { + "daily_data": [ + const chartdata = [ + { + date: 'Jan 22', + cache_hits: 10, + llm_api_calls: 2000 + }, + { + date: 'Jan 23', + cache_hits: 10, + llm_api_calls: 12 + }, + ], + "sum_cache_hits": 20, + "sum_llm_api_calls": 2012 + } + """ + from collections import defaultdict + + if start_date is None or end_date is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": "Please provide start_date and end_date"}, + ) + + start_date_obj = datetime.strptime(start_date, "%Y-%m-%d") + end_date_obj = datetime.strptime(end_date, "%Y-%m-%d") + + from litellm.proxy.proxy_server import llm_router, prisma_client + + try: + if prisma_client is None: + raise ValueError( + f"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" + ) + + sql_query = """ + SELECT + "api_key", + "call_type", + "model", + COUNT(*) AS total_rows, + SUM(CASE WHEN "cache_hit" = 'True' THEN 1 ELSE 0 END) AS cache_hit_true_rows + FROM "LiteLLM_SpendLogs" + WHERE + "startTime" BETWEEN $1::date AND $2::date + interval '1 day' + GROUP BY + "api_key", + "call_type", + "model" + """ + db_response = await prisma_client.db.query_raw( + sql_query, start_date_obj, end_date_obj + ) + + if db_response is None: + return [] + + return db_response + + except Exception as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": str(e)}, + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6d19766ad03..3befed9eaa3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -116,6 +116,9 @@ from litellm.exceptions import RejectedRequestError from litellm.integrations.slack_alerting import SlackAlerting, SlackAlertingArgs from litellm.llms.custom_httpx.httpx_handler import HTTPHandler from litellm.proxy._types import * +from litellm.proxy.analytics_endpoints.analytics_endpoints import ( + router as analytics_router, +) from litellm.proxy.auth.auth_checks import ( allowed_routes_check, common_checks, @@ -9139,3 +9142,4 @@ app.include_router(internal_user_router) app.include_router(team_router) app.include_router(spend_management_router) app.include_router(caching_router) +app.include_router(analytics_router) From 829ac3a0d099f20231c866d8aeeb590454293a25 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 21 Jun 2024 13:10:40 -0700 Subject: [PATCH 2/8] ui - add new cache hits page --- ui/litellm-dashboard/src/app/page.tsx | 9 + .../src/components/cache_dashboard.tsx | 182 ++++++++++++++++++ .../src/components/leftnav.tsx | 15 +- .../src/components/networking.tsx | 41 ++++ 4 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/cache_dashboard.tsx diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 3b70ef0aca6..ed9c98df7c6 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -15,6 +15,7 @@ import APIRef from "@/components/api_ref"; import ChatUI from "@/components/chat_ui"; import Sidebar from "../components/leftnav"; import Usage from "../components/usage"; +import CacheDashboard from "@/components/cache_dashboard"; import { jwtDecode } from "jwt-decode"; import { Typography } from "antd"; @@ -221,6 +222,14 @@ const CreateKeyPage = () => { publicPage={false} premiumUser={premiumUser} /> + ) : page == "caching" ? ( + ) : ( { + if (!date) return undefined; + return date.toISOString().split('T')[0]; + }; + +interface CachePageProps { + accessToken: string | null; + token: string | null; + userRole: string | null; + userID: string | null; + premiumUser: boolean; +} + + +const CacheDashboard: React.FC = ({ + accessToken, + token, + userRole, + userID, + premiumUser, +}) => { + const [filteredData, setFilteredData] = useState([]); + const [selectedApiKeys, setSelectedApiKeys] = useState([]); + const [selectedModels, setSelectedModels] = useState([]); + const [data, setData] = useState([]); + + const [dateValue, setDateValue] = useState({ + from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), + to: new Date(), + }); + + + useEffect(() => { + if (!accessToken || !dateValue) { + return; + } + const fetchData = async () => { + const response = await adminGlobalCacheActivity(accessToken, formatDateWithoutTZ(dateValue.from), formatDateWithoutTZ(dateValue.to)); + setData(response); + }; + fetchData(); + }, [accessToken]); + + const uniqueApiKeys = [...new Set(data.map((item) => item.api_key))]; + const uniqueModels = [...new Set(data.map((item) => item.model))]; + const uniqueCallTypes = [...new Set(data.map((item) => item.call_type))]; + + useEffect(() => { + console.log("DATA IN CACHE DASHBOARD", data); + let newData = data; + if (selectedApiKeys.length > 0) { + newData = newData.filter((item) => selectedApiKeys.includes(item.api_key)); + } + + if (selectedModels.length > 0) { + newData = newData.filter((item) => selectedModels.includes(item.model)); + } + + /* + Data looks like this + [{"api_key":"147dba2181f28914eea90eb484926c293cdcf7f5b5c9c3dd6a004d9e0f9fdb21","call_type":"acompletion","model":"llama3-8b-8192","total_rows":13,"cache_hit_true_rows":0}, + {"api_key":"8c23f021d0535c2e59abb7d83d0e03ccfb8db1b90e231ff082949d95df419e86","call_type":"None","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0}, + {"api_key":"88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b","call_type":"acompletion","model":"gpt-3.5-turbo","total_rows":19,"cache_hit_true_rows":0}, + {"api_key":"88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b","call_type":"aimage_generation","model":"","total_rows":3,"cache_hit_true_rows":0}, + {"api_key":"0ad4b3c03dcb6de0b5b8f761db798c6a8ae80be3fd1e2ea30c07ce6d5e3bf870","call_type":"None","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0}, + {"api_key":"034224b36e9769bc50e2190634abc3f97cad789b17ca80ac43b82f46cd5579b3","call_type":"","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0}, + {"api_key":"4f9c71cce0a2bb9a0b62ce6f0ebb3245b682702a8851d26932fa7e3b8ebfc755","call_type":"","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0}, + */ + + // What data we need for bar chat + // ui_data = [ + // { + // name: "Call Type", + // Cache hit: 20, + // LLM API requests: 10, + // } + // ] + + console.log("before processed data in cache dashboard", newData); + + const processedData = newData.reduce((acc, item) => { + console.log("Processing item:", item); + + if (!item.call_type) { + console.log("Item has no call_type:", item); + item.call_type = "Unknown"; + } + + const existingItem = acc.find(i => i.name === item.call_type); + if (existingItem) { + existingItem["LLM API requests"] += (item.total_rows || 0) - (item.cache_hit_true_rows || 0); + existingItem["Cache hit"] += item.cache_hit_true_rows || 0; + } else { + acc.push({ + name: item.call_type, + "LLM API requests": (item.total_rows || 0) - (item.cache_hit_true_rows || 0), + "Cache hit": item.cache_hit_true_rows || 0, + }); + } + return acc; + }, []); + + setFilteredData(processedData); + + console.log("PROCESSED DATA IN CACHE DASHBOARD", processedData); + + }, [selectedApiKeys, selectedModels, dateValue, data]); + + return ( + + API Activity Dashboard + Cache hits vs API requests broken down by call type + + + + + {uniqueApiKeys.map((key) => ( + + {key} + + ))} + + + + + {uniqueModels.map((model) => ( + + {model} + + ))} + + + + + + + + + + ); +}; + +export default CacheDashboard; \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index aa774088596..7a86fbcea62 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -85,22 +85,25 @@ const Sidebar: React.FC = ({ Budgets ) : null} - {userRole == "Admin" ? ( - setPage("general-settings")}> + setPage("caching")}> + Caching + + ) : null} + {userRole == "Admin" ? ( + setPage("general-settings")}> Router Settings ) : null} - {userRole == "Admin" ? ( - setPage("admin-panel")}> + setPage("admin-panel")}> Admin ) : null} - setPage("api_ref")}> + setPage("api_ref")}> API Reference - setPage("model-hub")}> + setPage("model-hub")}> Model Hub diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index e5371f22e4b..9060c339e13 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -1373,6 +1373,47 @@ export const adminGlobalActivity = async ( } }; +export const adminGlobalCacheActivity = async ( + accessToken: String, + startTime: String | undefined, + endTime: String | undefined +) => { + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/global/activity/cache_hits` + : `/global/activity/cache_hits`; + + if (startTime && endTime) { + url += `?start_date=${startTime}&end_date=${endTime}`; + } + + const requestOptions: { + method: string; + headers: { + Authorization: string; + }; + } = { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }; + + const response = await fetch(url, requestOptions); + + if (!response.ok) { + const errorData = await response.text(); + throw new Error("Network response was not ok"); + } + const data = await response.json(); + console.log(data); + return data; + } catch (error) { + console.error("Failed to fetch spend data:", error); + throw error; + } +}; + export const adminGlobalActivityPerModel = async ( accessToken: String, startTime: String | undefined, From b91854bdbb90eea5a7880038f1ccddad5d2694fb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 21 Jun 2024 13:43:47 -0700 Subject: [PATCH 3/8] ui - show leading cache indicators --- .../src/components/cache_dashboard.tsx | 87 ++++++++++++++++++- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/cache_dashboard.tsx b/ui/litellm-dashboard/src/components/cache_dashboard.tsx index b37b2862d72..e2a24d06cf1 100644 --- a/ui/litellm-dashboard/src/components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/cache_dashboard.tsx @@ -43,6 +43,9 @@ const CacheDashboard: React.FC = ({ const [selectedApiKeys, setSelectedApiKeys] = useState([]); const [selectedModels, setSelectedModels] = useState([]); const [data, setData] = useState([]); + const [cachedResponses, setCachedResponses] = useState(0); + const [cachedTokens, setCachedTokens] = useState(0); + const [cacheHitRatio, setCacheHitRatio] = useState("0"); const [dateValue, setDateValue] = useState({ from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), @@ -98,6 +101,9 @@ const CacheDashboard: React.FC = ({ console.log("before processed data in cache dashboard", newData); + let llm_api_requests = 0; + let cache_hits = 0; + let cached_tokens = 0; const processedData = newData.reduce((acc, item) => { console.log("Processing item:", item); @@ -105,20 +111,39 @@ const CacheDashboard: React.FC = ({ console.log("Item has no call_type:", item); item.call_type = "Unknown"; } + + + llm_api_requests += (item.total_rows || 0) - (item.cache_hit_true_rows || 0); + cache_hits += item.cache_hit_true_rows || 0; + cached_tokens += item.cached_completion_tokens || 0; const existingItem = acc.find(i => i.name === item.call_type); if (existingItem) { existingItem["LLM API requests"] += (item.total_rows || 0) - (item.cache_hit_true_rows || 0); existingItem["Cache hit"] += item.cache_hit_true_rows || 0; + existingItem["Cached Completion Tokens"] += item.cached_completion_tokens || 0; + existingItem["Generated Completion Tokens"] += item.generated_completion_tokens || 0; } else { acc.push({ name: item.call_type, "LLM API requests": (item.total_rows || 0) - (item.cache_hit_true_rows || 0), "Cache hit": item.cache_hit_true_rows || 0, + "Cached Completion Tokens": item.cached_completion_tokens || 0, + "Generated Completion Tokens": item.generated_completion_tokens || 0 }); } return acc; }, []); + + // set header cache statistics + setCachedResponses(cache_hits); + setCachedTokens(cached_tokens); + if (llm_api_requests > 0) { + let cache_hit_ratio = ((cache_hits / llm_api_requests) * 100).toFixed(2); + setCacheHitRatio(cache_hit_ratio); + } else { + setCacheHitRatio(0); + } setFilteredData(processedData); @@ -126,8 +151,8 @@ const CacheDashboard: React.FC = ({ }, [selectedApiKeys, selectedModels, dateValue, data]); - return ( - + return ( + API Activity Dashboard Cache hits vs API requests broken down by call type @@ -167,6 +192,45 @@ const CacheDashboard: React.FC = ({ +
+ +

+ Cache Hit Ratio +

+
+

+ {cacheHitRatio}% +

+ +
+ + +

+ Cache Hits +

+
+

+ {cachedResponses} +

+ +
+ + + +

+ Cached Tokens +

+
+

+ {cachedTokens} +

+ +
+ + +
+ + = ({ colors={["blue", "teal"]} yAxisWidth={48} /> - + + + + + + + + + + + ); }; From d30cdd82ccf459bcd40369f17300dd5e319e07df Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 21 Jun 2024 14:28:15 -0700 Subject: [PATCH 4/8] move caching up on left nav --- ui/litellm-dashboard/src/components/leftnav.tsx | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 7a86fbcea62..1fafedf18b0 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -79,17 +79,18 @@ const Sidebar: React.FC = ({ Logging & Alerts ) : null} - {userRole == "Admin" ? ( - setPage("budgets")}> - Budgets - - ) : null} - {userRole == "Admin" ? ( - setPage("caching")}> + setPage("caching")}> Caching ) : null} + + {userRole == "Admin" ? ( + setPage("budgets")}> + Budgets + + ) : null} + {userRole == "Admin" ? ( setPage("general-settings")}> Router Settings From e0a2717a1989e185dfa8661a2cd1ca286dc9dea8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 21 Jun 2024 14:36:28 -0700 Subject: [PATCH 5/8] ui - filter by selected date time --- .../src/components/cache_dashboard.tsx | 84 +++++++++++++------ 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/ui/litellm-dashboard/src/components/cache_dashboard.tsx b/ui/litellm-dashboard/src/components/cache_dashboard.tsx index e2a24d06cf1..e9d805042d2 100644 --- a/ui/litellm-dashboard/src/components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/cache_dashboard.tsx @@ -23,6 +23,17 @@ const formatDateWithoutTZ = (date: Date | undefined) => { return date.toISOString().split('T')[0]; }; + +function valueFormatterNumbers(number: number) { +const formatter = new Intl.NumberFormat('en-US', { + maximumFractionDigits: 0, + notation: 'compact', + compactDisplay: 'short', +}); + +return formatter.format(number); +} + interface CachePageProps { accessToken: string | null; token: string | null; @@ -43,8 +54,8 @@ const CacheDashboard: React.FC = ({ const [selectedApiKeys, setSelectedApiKeys] = useState([]); const [selectedModels, setSelectedModels] = useState([]); const [data, setData] = useState([]); - const [cachedResponses, setCachedResponses] = useState(0); - const [cachedTokens, setCachedTokens] = useState(0); + const [cachedResponses, setCachedResponses] = useState("0"); + const [cachedTokens, setCachedTokens] = useState("0"); const [cacheHitRatio, setCacheHitRatio] = useState("0"); const [dateValue, setDateValue] = useState({ @@ -64,9 +75,31 @@ const CacheDashboard: React.FC = ({ fetchData(); }, [accessToken]); - const uniqueApiKeys = [...new Set(data.map((item) => item.api_key))]; - const uniqueModels = [...new Set(data.map((item) => item.model))]; - const uniqueCallTypes = [...new Set(data.map((item) => item.call_type))]; + const uniqueApiKeys = [...new Set(data.map((item) => item?.api_key ? item.api_key : ""))]; + const uniqueModels = [...new Set(data.map((item) => item?.model ? item.model : ""))]; + const uniqueCallTypes = [...new Set(data.map((item) => item?.call_type ? item.call_type : ""))]; + + + const updateCachingData = async (startTime: Date | undefined, endTime: Date | undefined) => { + if (!startTime || !endTime || !accessToken) { + return; + } + + // the endTime put it to the last hour of the selected date + endTime.setHours(23, 59, 59, 999); + + // startTime put it to the first hour of the selected date + startTime.setHours(0, 0, 0, 0); + + let new_cache_data = await adminGlobalCacheActivity( + accessToken, + formatDateWithoutTZ(startTime), + formatDateWithoutTZ(endTime) + ) + + setData(new_cache_data); + + } useEffect(() => { console.log("DATA IN CACHE DASHBOARD", data); @@ -136,8 +169,8 @@ const CacheDashboard: React.FC = ({ }, []); // set header cache statistics - setCachedResponses(cache_hits); - setCachedTokens(cached_tokens); + setCachedResponses(valueFormatterNumbers(cache_hits)); + setCachedTokens(valueFormatterNumbers(cached_tokens)); if (llm_api_requests > 0) { let cache_hit_ratio = ((cache_hits / llm_api_requests) * 100).toFixed(2); setCacheHitRatio(cache_hit_ratio); @@ -152,10 +185,7 @@ const CacheDashboard: React.FC = ({ }, [selectedApiKeys, selectedModels, dateValue, data]); return ( - - API Activity Dashboard - Cache hits vs API requests broken down by call type - + = ({ { + setDateValue(value); + updateCachingData(value.from, value.to); + }} selectPlaceholder="Select date range" /> @@ -230,26 +264,28 @@ const CacheDashboard: React.FC = ({ - + Cache Hits vs API Requests - - - + Cached Completion Tokens vs Generated Completion Tokens + + From 35c07306df319a25757f679005aec5d25030cd6b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 21 Jun 2024 14:36:38 -0700 Subject: [PATCH 6/8] show correct key aliases on ui --- .../analytics_endpoints.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/analytics_endpoints/analytics_endpoints.py b/litellm/proxy/analytics_endpoints/analytics_endpoints.py index ebb7d43acfd..54ac8bc48fb 100644 --- a/litellm/proxy/analytics_endpoints/analytics_endpoints.py +++ b/litellm/proxy/analytics_endpoints/analytics_endpoints.py @@ -74,18 +74,24 @@ async def get_global_activity( sql_query = """ SELECT - "api_key", - "call_type", - "model", + CASE + WHEN vt."key_alias" IS NOT NULL THEN vt."key_alias" + ELSE 'Unnamed Key' + END AS api_key, + sl."call_type", + sl."model", COUNT(*) AS total_rows, - SUM(CASE WHEN "cache_hit" = 'True' THEN 1 ELSE 0 END) AS cache_hit_true_rows - FROM "LiteLLM_SpendLogs" + SUM(CASE WHEN sl."cache_hit" = 'True' THEN 1 ELSE 0 END) AS cache_hit_true_rows, + SUM(CASE WHEN sl."cache_hit" = 'True' THEN sl."completion_tokens" ELSE 0 END) AS cached_completion_tokens, + SUM(CASE WHEN sl."cache_hit" != 'True' THEN sl."completion_tokens" ELSE 0 END) AS generated_completion_tokens + FROM "LiteLLM_SpendLogs" sl + LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token" WHERE - "startTime" BETWEEN $1::date AND $2::date + interval '1 day' + sl."startTime" BETWEEN $1::date AND $2::date + interval '1 day' GROUP BY - "api_key", - "call_type", - "model" + vt."key_alias", + sl."call_type", + sl."model" """ db_response = await prisma_client.db.query_raw( sql_query, start_date_obj, end_date_obj From 097474502a338dab7745a8478fd59a7c8f250372 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 21 Jun 2024 15:00:26 -0700 Subject: [PATCH 7/8] ui - fix linting errors --- .../src/components/cache_dashboard.tsx | 47 ++++++++++++++----- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/ui/litellm-dashboard/src/components/cache_dashboard.tsx b/ui/litellm-dashboard/src/components/cache_dashboard.tsx index e9d805042d2..d9c48f960b9 100644 --- a/ui/litellm-dashboard/src/components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/cache_dashboard.tsx @@ -42,6 +42,29 @@ interface CachePageProps { premiumUser: boolean; } +interface cacheDataItem { + api_key: string; + model: string; + cache_hit_true_rows: number; + cached_completion_tokens: number; + total_rows: number; + generated_completion_tokens: number; + call_type: string; + + // Add other properties as needed + } + + +interface uiData { + "name": string; + "LLM API requests": number; + "Cache hit": number; + "Cached Completion Tokens": number; + "Generated Completion Tokens": number; + +} + + const CacheDashboard: React.FC = ({ accessToken, @@ -50,10 +73,10 @@ const CacheDashboard: React.FC = ({ userID, premiumUser, }) => { - const [filteredData, setFilteredData] = useState([]); - const [selectedApiKeys, setSelectedApiKeys] = useState([]); - const [selectedModels, setSelectedModels] = useState([]); - const [data, setData] = useState([]); + const [filteredData, setFilteredData] = useState([]); + const [selectedApiKeys, setSelectedApiKeys] = useState([]); + const [selectedModels, setSelectedModels] = useState([]); + const [data, setData] = useState([]); const [cachedResponses, setCachedResponses] = useState("0"); const [cachedTokens, setCachedTokens] = useState("0"); const [cacheHitRatio, setCacheHitRatio] = useState("0"); @@ -75,9 +98,9 @@ const CacheDashboard: React.FC = ({ fetchData(); }, [accessToken]); - const uniqueApiKeys = [...new Set(data.map((item) => item?.api_key ? item.api_key : ""))]; - const uniqueModels = [...new Set(data.map((item) => item?.model ? item.model : ""))]; - const uniqueCallTypes = [...new Set(data.map((item) => item?.call_type ? item.call_type : ""))]; + const uniqueApiKeys = Array.from(new Set(data.map((item) => item?.api_key ?? ""))); + const uniqueModels = Array.from(new Set(data.map((item) => item?.model ?? ""))); + const uniqueCallTypes = Array.from(new Set(data.map((item) => item?.call_type ?? ""))); const updateCachingData = async (startTime: Date | undefined, endTime: Date | undefined) => { @@ -103,7 +126,7 @@ const CacheDashboard: React.FC = ({ useEffect(() => { console.log("DATA IN CACHE DASHBOARD", data); - let newData = data; + let newData: cacheDataItem[] = data; if (selectedApiKeys.length > 0) { newData = newData.filter((item) => selectedApiKeys.includes(item.api_key)); } @@ -137,7 +160,7 @@ const CacheDashboard: React.FC = ({ let llm_api_requests = 0; let cache_hits = 0; let cached_tokens = 0; - const processedData = newData.reduce((acc, item) => { + const processedData = newData.reduce((acc: uiData[], item) => { console.log("Processing item:", item); if (!item.call_type) { @@ -175,7 +198,7 @@ const CacheDashboard: React.FC = ({ let cache_hit_ratio = ((cache_hits / llm_api_requests) * 100).toFixed(2); setCacheHitRatio(cache_hit_ratio); } else { - setCacheHitRatio(0); + setCacheHitRatio("0"); } setFilteredData(processedData); @@ -271,7 +294,7 @@ const CacheDashboard: React.FC = ({ index="name" valueFormatter={valueFormatterNumbers} categories={["LLM API requests", "Cache hit"]} - colors={["blue", "teal"]} + colors={["sky", "teal"]} yAxisWidth={48} /> @@ -282,7 +305,7 @@ const CacheDashboard: React.FC = ({ index="name" valueFormatter={valueFormatterNumbers} categories={["Generated Completion Tokens", "Cached Completion Tokens"]} - colors={["blue", "teal"]} + colors={["sky", "teal"]} yAxisWidth={48} /> From 07f34ac25641c2342271a980103b14311aa3373f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 21 Jun 2024 15:26:30 -0700 Subject: [PATCH 8/8] ui - show cache hits --- ui/litellm-dashboard/src/components/cache_dashboard.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/cache_dashboard.tsx b/ui/litellm-dashboard/src/components/cache_dashboard.tsx index d9c48f960b9..783f24ed1b8 100644 --- a/ui/litellm-dashboard/src/components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/cache_dashboard.tsx @@ -287,7 +287,7 @@ const CacheDashboard: React.FC = ({ - Cache Hits vs API Requests + Cache Hits vs API Requests = ({ yAxisWidth={48} /> - Cached Completion Tokens vs Generated Completion Tokens + Cached Completion Tokens vs Generated Completion Tokens