From 38cf04cc384dfa62cbdee09b323bf4c4ec025cc9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 1 May 2024 08:26:44 -0700 Subject: [PATCH 1/5] fix tool tip --- .../src/components/model_dashboard.tsx | 108 +++++++++++++----- 1 file changed, 78 insertions(+), 30 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 40bcdf67529..af49f5a2607 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -475,38 +475,40 @@ const handleEditSubmit = async (formValues: Record) => { setModelExceptions(modelExceptionsResponse.data); setAllExceptions(modelExceptionsResponse.exception_types); - // let successdeploymentToSuccess: Record = {}; - // for (let i = 0; i < modelMetricsResponse.length; i++) { - // let element = modelMetricsResponse[i]; - // let _model_name = element.model; - // let _num_requests = element.num_requests; - // successdeploymentToSuccess[_model_name] = _num_requests - // } - // console.log("successdeploymentToSuccess:", successdeploymentToSuccess) + + let modelMetricsData = modelMetricsResponse.data; + let successdeploymentToSuccess: Record = {}; + for (let i = 0; i < modelMetricsData.length; i++) { + let element = modelMetricsData[i]; + let _model_name = element.model; + let _num_requests = element.num_requests; + successdeploymentToSuccess[_model_name] = _num_requests + } + console.log("successdeploymentToSuccess:", successdeploymentToSuccess) - // let failureTableData = []; - // let _failureData = modelExceptionsResponse.data; - // for (let i = 0; i < _failureData.length; i++) { - // const model = _failureData[i]; - // let _model_name = model.model; - // let total_exceptions = model.total_exceptions; - // let total_Requests = successdeploymentToSuccess[_model_name]; - // if (total_Requests == null) { - // total_Requests = 0 - // } - // let _data = { - // model: _model_name, - // total_exceptions: total_exceptions, - // total_Requests: total_Requests, - // failure_rate: total_Requests / total_exceptions - // } - // failureTableData.push(_data); - // // sort failureTableData by failure_rate - // failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); + let failureTableData = []; + let _failureData = modelExceptionsResponse.data; + for (let i = 0; i < _failureData.length; i++) { + const model = _failureData[i]; + let _model_name = model.model; + let total_exceptions = model.total_exceptions; + let total_Requests = successdeploymentToSuccess[_model_name]; + if (total_Requests == null) { + total_Requests = 0 + } + let _data = { + model: _model_name, + total_exceptions: total_exceptions, + total_Requests: total_Requests, + failure_rate: total_Requests / total_exceptions + } + failureTableData.push(_data); + // sort failureTableData by failure_rate + failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); - // setFailureTableData(failureTableData); - // console.log("failureTableData:", failureTableData); - // } + setFailureTableData(failureTableData); + console.log("failureTableData:", failureTableData); + } } catch (error) { console.error("There was an error fetching the model data", error); @@ -691,6 +693,50 @@ const handleEditSubmit = async (formValues: Record) => { } } + const customTooltip = (props) => { + const { payload, active } = props; + if (!active || !payload) return null; + + // Sort the payload array by category.value in descending order + let sortedPayload = payload.sort((a, b) => b.value - a.value); + + console.log("sortedPayload:", sortedPayload); + + // only show the top 5, the 6th one should be called "X other categories" depending on how many categories were not shown + if (sortedPayload.length > 5 ) { + let remainingItems = sortedPayload.length - 5; + sortedPayload = sortedPayload.slice(0, 5); + + + sortedPayload.push({ + dataKey: `${remainingItems} other deployments`, + value: payload.slice(5).reduce((acc, curr) => acc + curr.value, 0), + color: "gray", + }) + } + + + return ( +
+ {sortedPayload.map((category, idx) => ( +
+ + {/*
*/} + +

{category.dataKey}

+

+ {category.value.toFixed(5)} +

+
+ //
+ + ))} +
+ ); + }; + const getPlaceholder = (selectedProvider: string): string => { @@ -1073,12 +1119,14 @@ const handleEditSubmit = async (formValues: Record) => { { modelMetrics && modelMetricsCategories && ( )} From 94b98f5c4e5538b567533a9b50ccb81fe0559f15 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 1 May 2024 08:27:01 -0700 Subject: [PATCH 2/5] clean up model latency metrics --- litellm/proxy/proxy_server.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0dd4fbb43f2..22717b92575 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7540,15 +7540,24 @@ async def model_metrics( endTime = endTime or datetime.now() sql_query = """ - SELECT api_base, model, DATE_TRUNC('day', "startTime")::DATE AS day, - CASE - WHEN SUM(total_tokens) = 0 THEN 0 - ELSE AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) / SUM(total_tokens) - END AS avg_latency_per_token - FROM "LiteLLM_SpendLogs" - WHERE "startTime" >= NOW() - INTERVAL '30 days' AND "model" = $1 - GROUP BY api_base, model, day - ORDER BY avg_latency_per_token DESC; + SELECT + api_base, + model, + DATE_TRUNC('day', "startTime")::DATE AS day, + AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) / SUM(total_tokens) AS avg_latency_per_token + FROM + "LiteLLM_SpendLogs" + WHERE + "startTime" >= NOW() - INTERVAL '30 days' + AND "model" = $1 + GROUP BY + api_base, + model, + day + HAVING + SUM(total_tokens) > 0 + ORDER BY + avg_latency_per_token DESC; """ _all_api_bases = set() db_response = await prisma_client.db.query_raw( @@ -7566,6 +7575,8 @@ async def model_metrics( _combined_model_name = str(_model) if "https://" in _api_base: _combined_model_name = str(_api_base) + if "/openai/" in _combined_model_name: + _combined_model_name = _combined_model_name.split("/openai/")[0] _all_api_bases.add(_combined_model_name) _daily_entries[_day][_combined_model_name] = _avg_latency_per_token @@ -7587,7 +7598,7 @@ async def model_metrics( for day in _daily_entries: entry = {"date": str(day)} for model_key, latency in _daily_entries[day].items(): - entry[model_key] = latency.__round__(5) + entry[model_key] = round(latency, 8) response.append(entry) return { From e3229d0468a256482b31c041439a1419c3caeea6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 1 May 2024 08:56:21 -0700 Subject: [PATCH 3/5] fix tool tip --- .../src/components/model_dashboard.tsx | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index af49f5a2607..81d0aea26f5 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -693,45 +693,40 @@ const handleEditSubmit = async (formValues: Record) => { } } - const customTooltip = (props) => { + const customTooltip = (props: any) => { const { payload, active } = props; if (!active || !payload) return null; + // Extract the date from the first item in the payload array + const date = payload[0]?.payload?.date; + // Sort the payload array by category.value in descending order - let sortedPayload = payload.sort((a, b) => b.value - a.value); - - console.log("sortedPayload:", sortedPayload); - - // only show the top 5, the 6th one should be called "X other categories" depending on how many categories were not shown - if (sortedPayload.length > 5 ) { + let sortedPayload = payload.sort((a: any, b: any) => b.value - a.value); + + // Only show the top 5, the 6th one should be called "X other categories" depending on how many categories were not shown + if (sortedPayload.length > 5) { let remainingItems = sortedPayload.length - 5; sortedPayload = sortedPayload.slice(0, 5); - - sortedPayload.push({ dataKey: `${remainingItems} other deployments`, - value: payload.slice(5).reduce((acc, curr) => acc + curr.value, 0), + value: payload.slice(5).reduce((acc: number, curr: any) => acc + curr.value, 0), color: "gray", - }) + }); } - return (
- {sortedPayload.map((category, idx) => ( -
- - {/*
*/} - + {date &&

Date: {date}

} + {sortedPayload.map((category: any, idx: number) => ( +
+
+

{category.dataKey}

-

- {category.value.toFixed(5)} -

-
- //
- +
+

+ {category.value.toFixed(5)} +

+
))}
); From e3ce97d33e49d9995b6d42ec659aa034412ccd4c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 1 May 2024 08:59:05 -0700 Subject: [PATCH 4/5] ui - sort the dropdown --- .../src/components/model_dashboard.tsx | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 81d0aea26f5..8f12f9d6808 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -448,6 +448,9 @@ const handleEditSubmit = async (formValues: Record) => { } console.log("all_model_groups:", all_model_groups) let _array_model_groups = Array.from(all_model_groups) + // sort _array_model_groups alphabetically + _array_model_groups = _array_model_groups.sort(); + setAvailableModelGroups(_array_model_groups); const modelMetricsResponse = await modelMetricsCall( @@ -1090,14 +1093,7 @@ const handleEditSubmit = async (formValues: Record) => {

View how requests were load balanced within a model group