diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0dd4fbb43f2..22717b92575 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7540,15 +7540,24 @@ async def model_metrics( endTime = endTime or datetime.now() sql_query = """ - SELECT api_base, model, DATE_TRUNC('day', "startTime")::DATE AS day, - CASE - WHEN SUM(total_tokens) = 0 THEN 0 - ELSE AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) / SUM(total_tokens) - END AS avg_latency_per_token - FROM "LiteLLM_SpendLogs" - WHERE "startTime" >= NOW() - INTERVAL '30 days' AND "model" = $1 - GROUP BY api_base, model, day - ORDER BY avg_latency_per_token DESC; + SELECT + api_base, + model, + DATE_TRUNC('day', "startTime")::DATE AS day, + AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) / SUM(total_tokens) AS avg_latency_per_token + FROM + "LiteLLM_SpendLogs" + WHERE + "startTime" >= NOW() - INTERVAL '30 days' + AND "model" = $1 + GROUP BY + api_base, + model, + day + HAVING + SUM(total_tokens) > 0 + ORDER BY + avg_latency_per_token DESC; """ _all_api_bases = set() db_response = await prisma_client.db.query_raw( @@ -7566,6 +7575,8 @@ async def model_metrics( _combined_model_name = str(_model) if "https://" in _api_base: _combined_model_name = str(_api_base) + if "/openai/" in _combined_model_name: + _combined_model_name = _combined_model_name.split("/openai/")[0] _all_api_bases.add(_combined_model_name) _daily_entries[_day][_combined_model_name] = _avg_latency_per_token @@ -7587,7 +7598,7 @@ async def model_metrics( for day in _daily_entries: entry = {"date": str(day)} for model_key, latency in _daily_entries[day].items(): - entry[model_key] = latency.__round__(5) + entry[model_key] = round(latency, 8) response.append(entry) return { diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 40bcdf67529..12e650ac72b 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -448,6 +448,9 @@ const handleEditSubmit = async (formValues: Record) => { } console.log("all_model_groups:", all_model_groups) let _array_model_groups = Array.from(all_model_groups) + // sort _array_model_groups alphabetically + _array_model_groups = _array_model_groups.sort(); + setAvailableModelGroups(_array_model_groups); const modelMetricsResponse = await modelMetricsCall( @@ -475,38 +478,40 @@ const handleEditSubmit = async (formValues: Record) => { setModelExceptions(modelExceptionsResponse.data); setAllExceptions(modelExceptionsResponse.exception_types); - // let successdeploymentToSuccess: Record = {}; - // for (let i = 0; i < modelMetricsResponse.length; i++) { - // let element = modelMetricsResponse[i]; - // let _model_name = element.model; - // let _num_requests = element.num_requests; - // successdeploymentToSuccess[_model_name] = _num_requests - // } - // console.log("successdeploymentToSuccess:", successdeploymentToSuccess) + + let modelMetricsData = modelMetricsResponse.data; + let successdeploymentToSuccess: Record = {}; + for (let i = 0; i < modelMetricsData.length; i++) { + let element = modelMetricsData[i]; + let _model_name = element.model; + let _num_requests = element.num_requests; + successdeploymentToSuccess[_model_name] = _num_requests + } + console.log("successdeploymentToSuccess:", successdeploymentToSuccess) - // let failureTableData = []; - // let _failureData = modelExceptionsResponse.data; - // for (let i = 0; i < _failureData.length; i++) { - // const model = _failureData[i]; - // let _model_name = model.model; - // let total_exceptions = model.total_exceptions; - // let total_Requests = successdeploymentToSuccess[_model_name]; - // if (total_Requests == null) { - // total_Requests = 0 - // } - // let _data = { - // model: _model_name, - // total_exceptions: total_exceptions, - // total_Requests: total_Requests, - // failure_rate: total_Requests / total_exceptions - // } - // failureTableData.push(_data); - // // sort failureTableData by failure_rate - // failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); + let failureTableData = []; + let _failureData = modelExceptionsResponse.data; + for (let i = 0; i < _failureData.length; i++) { + const model = _failureData[i]; + let _model_name = model.model; + let total_exceptions = model.total_exceptions; + let total_Requests = successdeploymentToSuccess[_model_name]; + if (total_Requests == null) { + total_Requests = 0 + } + let _data = { + model: _model_name, + total_exceptions: total_exceptions, + total_Requests: total_Requests, + failure_rate: total_Requests / total_exceptions + } + failureTableData.push(_data); + // sort failureTableData by failure_rate + failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); - // setFailureTableData(failureTableData); - // console.log("failureTableData:", failureTableData); - // } + setFailureTableData(failureTableData); + console.log("failureTableData:", failureTableData); + } } catch (error) { console.error("There was an error fetching the model data", error); @@ -691,6 +696,45 @@ const handleEditSubmit = async (formValues: Record) => { } } + const customTooltip = (props: any) => { + const { payload, active } = props; + if (!active || !payload) return null; + + // Extract the date from the first item in the payload array + const date = payload[0]?.payload?.date; + + // Sort the payload array by category.value in descending order + let sortedPayload = payload.sort((a: any, b: any) => b.value - a.value); + + // Only show the top 5, the 6th one should be called "X other categories" depending on how many categories were not shown + if (sortedPayload.length > 5) { + let remainingItems = sortedPayload.length - 5; + sortedPayload = sortedPayload.slice(0, 5); + sortedPayload.push({ + dataKey: `${remainingItems} other deployments`, + value: payload.slice(5).reduce((acc: number, curr: any) => acc + curr.value, 0), + color: "gray", + }); + } + + return ( +
+ {date &&

Date: {date}

} + {sortedPayload.map((category: any, idx: number) => ( +
+
+
+

{category.dataKey}

+
+

+ {category.value.toFixed(5)} +

+
+ ))} +
+ ); + }; + const getPlaceholder = (selectedProvider: string): string => { @@ -1049,14 +1093,7 @@ const handleEditSubmit = async (formValues: Record) => {

View how requests were load balanced within a model group