From ac6809e9df77db7875611066fba742b14701f47f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Apr 2024 20:42:15 -0700 Subject: [PATCH 1/5] ui - filter by time and deployments --- litellm/proxy/proxy_server.py | 62 +++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 568121dbad1..eda78189552 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7323,8 +7323,11 @@ async def model_info_v2( ) async def model_metrics( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + _selected_model_group: Optional[str] = None, + startTime: Optional[datetime] = datetime.now() - timedelta(days=30), + endTime: Optional[datetime] = datetime.now(), ): - global prisma_client + global prisma_client, llm_router if prisma_client is None: raise ProxyException( message="Prisma Client is not initialized", @@ -7332,24 +7335,49 @@ async def model_metrics( param="None", code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + if _selected_model_group and llm_router is not None: + _model_list = llm_router.get_model_list() + _relevant_api_bases = [] + for model in _model_list: + if model["model_name"] == _selected_model_group: + _litellm_params = model["litellm_params"] + _api_base = _litellm_params.get("api_base", "") + _relevant_api_bases.append(_api_base) - sql_query = """ - SELECT - CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base, - COUNT(*) AS num_requests, - AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) AS avg_latency_seconds - FROM - "LiteLLM_SpendLogs" - WHERE - "startTime" >= NOW() - INTERVAL '10000 hours' - GROUP BY - CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END - ORDER BY - num_requests DESC - LIMIT 50; - """ + sql_query = """ + SELECT + CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base, + COUNT(*) AS num_requests, + AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) AS avg_latency_seconds + FROM "LiteLLM_SpendLogs" + WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp + AND api_base = ANY($3) + GROUP BY CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END + ORDER BY num_requests DESC + LIMIT 50; + """ - db_response = await prisma_client.db.query_raw(query=sql_query) + db_response = await prisma_client.db.query_raw( + sql_query, startTime, endTime, _relevant_api_bases + ) + else: + + sql_query = """ + SELECT + CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base, + COUNT(*) AS num_requests, + AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) AS avg_latency_seconds + FROM + "LiteLLM_SpendLogs" + WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp + GROUP BY + CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END + ORDER BY + num_requests DESC + LIMIT 50; + """ + + db_response = await prisma_client.db.query_raw(sql_query, startTime, endTime) response: List[dict] = [] if response is not None: # loop through all models From 9017d9bb8197d80436ef425ad2b445481b3e524a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Apr 2024 22:03:20 -0700 Subject: [PATCH 2/5] backend allow filtering by model_group --- litellm/proxy/proxy_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index eda78189552..5af18c0ee6a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7343,6 +7343,7 @@ async def model_metrics( _litellm_params = model["litellm_params"] _api_base = _litellm_params.get("api_base", "") _relevant_api_bases.append(_api_base) + _relevant_api_bases.append(_api_base + "/openai/") sql_query = """ SELECT From 6d828b478d84dc2b1c9d438fd48d133a1d1c24c6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Apr 2024 22:03:59 -0700 Subject: [PATCH 3/5] ui - allow filtering by model group --- ui/litellm-dashboard/src/components/usage.tsx | 77 ++++++++++++++++++- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/components/usage.tsx index d96184b9a86..33f0c844540 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/components/usage.tsx @@ -3,7 +3,7 @@ import { BarChart, BarList, Card, Title, Table, TableHead, TableHeaderCell, Tabl import React, { useState, useEffect } from "react"; import ViewUserSpend from "./view_user_spend"; -import { Grid, Col, Text, LineChart, TabPanel, TabPanels, TabGroup, TabList, Tab } from "@tremor/react"; +import { Grid, Col, Text, LineChart, TabPanel, TabPanels, TabGroup, TabList, Tab, Select, SelectItem } from "@tremor/react"; import { userSpendLogsCall, keyInfoCall, @@ -13,6 +13,8 @@ import { teamSpendLogsCall, tagsSpendLogsCall, modelMetricsCall, + modelAvailableCall, + modelInfoCall, } from "./networking"; import { start } from "repl"; @@ -146,6 +148,8 @@ const UsagePage: React.FC = ({ const [totalSpendPerTeam, setTotalSpendPerTeam] = useState([]); const [modelMetrics, setModelMetrics] = useState([]); const [modelLatencyMetrics, setModelLatencyMetrics] = useState([]); + const [modelGroups, setModelGroups] = useState([]); + const [selectedModelGroup, setSelectedModelGroup] = useState(null); const firstDay = new Date( currentDate.getFullYear(), @@ -227,7 +231,25 @@ const UsagePage: React.FC = ({ const top_tags = await tagsSpendLogsCall(accessToken); setTopTagsData(top_tags.top_10_tags); - + // get model groups + const _model_groups = await modelInfoCall(accessToken, userID, userRole); + let model_groups = _model_groups.data; + console.log("model groups in model dashboard", model_groups); + + let available_model_groups = []; + // loop through each model in model_group, access litellm_params and only inlclude the model if model["litellm_params"]["model"] startswith "azure/" + for (let i = 0; i < model_groups.length; i++) { + let model = model_groups[i]; + console.log("model check", model); + let model_group = model["litellm_params"]["model"]; + console.log("model group", model_group); + if (model_group.startsWith("azure/")) { + available_model_groups.push(model["model_name"]); + } + } + setModelGroups(available_model_groups); + + } else if (userRole == "App Owner") { await userSpendLogsCall( accessToken, @@ -268,7 +290,8 @@ const UsagePage: React.FC = ({ const modelMetricsResponse = await modelMetricsCall( accessToken, userID, - userRole + userRole, + null ); console.log("Model metrics response:", modelMetricsResponse); @@ -288,6 +311,31 @@ const UsagePage: React.FC = ({ } }, [accessToken, token, userRole, userID, startTime, endTime]); + + const updateModelMetrics = async (modelGroup: string | null) => { + console.log("Updating model metrics for group:", modelGroup); + if (!accessToken || !userID || !userRole) { + return + } + setSelectedModelGroup(modelGroup); // If you want to store the selected model group in state + + + try { + const modelMetricsResponse = await modelMetricsCall(accessToken, userID, userRole, modelGroup); + console.log("Model metrics response:", modelMetricsResponse); + + // Assuming modelMetricsResponse now contains the metric data for the specified model group + const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_seconds - a.avg_latency_seconds); + console.log("Sorted by latency:", sortedByLatency); + + setModelMetrics(modelMetricsResponse); + setModelLatencyMetrics(sortedByLatency); + } catch (error) { + console.error("Failed to fetch model metrics", error); + } + } + + return (
= ({ - + Spend Per Tag - Last 30 Days @@ -445,6 +493,27 @@ const UsagePage: React.FC = ({ + + Number Requests per Model Date: Tue, 23 Apr 2024 22:04:12 -0700 Subject: [PATCH 4/5] ui filter by model group --- ui/litellm-dashboard/src/components/networking.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 96b6246f55f..0f44d51b574 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -403,13 +403,17 @@ export const modelInfoCall = async ( export const modelMetricsCall = async ( accessToken: String, userID: String, - userRole: String + userRole: String, + modelGroup: String | null, ) => { /** * Get all models on proxy */ try { let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics` : `/model/metrics`; + if (modelGroup) { + url = `${url}?_selected_model_group=${modelGroup}` + } // message.info("Requesting model data"); const response = await fetch(url, { method: "GET", From b2fdc3fc769405b6c18b88a1236a6735b9356806 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Apr 2024 22:08:59 -0700 Subject: [PATCH 5/5] ui - show how models were load balanced --- ui/litellm-dashboard/src/components/usage.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/components/usage.tsx index 33f0c844540..0caeaccc468 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/components/usage.tsx @@ -493,6 +493,10 @@ const UsagePage: React.FC = ({ + Filter By Model Group +

View how requests were load balanced within a model group

+

(Beta feature) only supported for Azure Model Groups

+