From 5041b924acee874d7b2ca785c57d2fbb999e4750 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 1 May 2024 17:23:37 -0700 Subject: [PATCH] ui - show slow responses on admin UI --- .../src/components/model_dashboard.tsx | 96 ++++++++++--------- .../src/components/networking.tsx | 40 ++++++++ 2 files changed, 93 insertions(+), 43 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index c9fcdf228bf..f8af6f5bf98 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -18,7 +18,7 @@ import { } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react"; import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react"; -import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall } from "./networking"; +import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall } from "./networking"; import { BarChart, AreaChart } from "@tremor/react"; import { Button as Button2, @@ -205,6 +205,7 @@ const ModelDashboard: React.FC = ({ const [modelExceptions, setModelExceptions] = useState([]); const [allExceptions, setAllExceptions] = useState([]); const [failureTableData, setFailureTableData] = useState([]); + const [slowResponsesData, setSlowResponsesData] = useState([]); const EditModelModal: React.FC = ({ visible, onCancel, model, onSubmit }) => { const [form] = Form.useForm(); @@ -479,39 +480,51 @@ const handleEditSubmit = async (formValues: Record) => { setAllExceptions(modelExceptionsResponse.exception_types); - let modelMetricsData = modelMetricsResponse.data; - let successdeploymentToSuccess: Record = {}; - for (let i = 0; i < modelMetricsData.length; i++) { - let element = modelMetricsData[i]; - let _model_name = element.model; - let _num_requests = element.num_requests; - successdeploymentToSuccess[_model_name] = _num_requests - } - console.log("successdeploymentToSuccess:", successdeploymentToSuccess) + const slowResponses = await modelMetricsSlowResponsesCall( + accessToken, + userID, + userRole, + null + ) + + console.log("slowResponses:", slowResponses) + + setSlowResponsesData(slowResponses); + + + // let modelMetricsData = modelMetricsResponse.data; + // let successdeploymentToSuccess: Record = {}; + // for (let i = 0; i < modelMetricsData.length; i++) { + // let element = modelMetricsData[i]; + // let _model_name = element.model; + // let _num_requests = element.num_requests; + // successdeploymentToSuccess[_model_name] = _num_requests + // } + // console.log("successdeploymentToSuccess:", successdeploymentToSuccess) - let failureTableData = []; - let _failureData = modelExceptionsResponse.data; - for (let i = 0; i < _failureData.length; i++) { - const model = _failureData[i]; - let _model_name = model.model; - let total_exceptions = model.total_exceptions; - let total_Requests = successdeploymentToSuccess[_model_name]; - if (total_Requests == null) { - total_Requests = 0 - } - let _data = { - model: _model_name, - total_exceptions: total_exceptions, - total_Requests: total_Requests, - failure_rate: total_Requests / total_exceptions - } - failureTableData.push(_data); - // sort failureTableData by failure_rate - failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); + // let failureTableData = []; + // let _failureData = modelExceptionsResponse.data; + // for (let i = 0; i < _failureData.length; i++) { + // const model = _failureData[i]; + // let _model_name = model.model; + // let total_exceptions = model.total_exceptions; + // let total_Requests = successdeploymentToSuccess[_model_name]; + // if (total_Requests == null) { + // total_Requests = 0 + // } + // let _data = { + // model: _model_name, + // total_exceptions: total_exceptions, + // total_Requests: total_Requests, + // failure_rate: total_Requests / total_exceptions + // } + // failureTableData.push(_data); + // // sort failureTableData by failure_rate + // failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); - setFailureTableData(failureTableData); - console.log("failureTableData:", failureTableData); - } + // setFailureTableData(failureTableData); + // console.log("failureTableData:", failureTableData); + // } } catch (error) { console.error("There was an error fetching the model data", error); @@ -1126,8 +1139,6 @@ const handleEditSubmit = async (formValues: Record) => { /> )} - - @@ -1135,20 +1146,19 @@ const handleEditSubmit = async (formValues: Record) => { - Model - Success Requests - Error Requests - Failure % + Deployment + Success Responses + Slow Responses - {failureTableData.map((metric, idx) => ( + {slowResponsesData.map((metric, idx) => ( - {metric.model} - {metric.total_Requests} - {metric.total_exceptions} - {metric.failure_rate}% + {metric.api_base} + {metric.total_count} + {metric.slow_count} + ))} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index ed17b7daa63..35880949b2d 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -475,6 +475,46 @@ export const modelMetricsCall = async ( }; + +export const modelMetricsSlowResponsesCall = async ( + accessToken: String, + userID: String, + userRole: String, + modelGroup: String | null, +) => { + /** + * Get all models on proxy + */ + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics/slow_responses` : `/model/metrics/slow_responses`; + if (modelGroup) { + url = `${url}?_selected_model_group=${modelGroup}` + } + // message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData, 20); + throw new Error("Network response was not ok"); + } + const data = await response.json(); + // message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + + export const modelExceptionsCall = async ( accessToken: String, userID: String,