Merge pull request #3259 from BerriAI/litellm_filter_metrics_model_group

UI - filter metrics by model group

litellm/proxy/proxy_server.py

@@ -7323,8 +7323,11 @@ async def model_info_v2(
 )
 async def model_metrics(
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    _selected_model_group: Optional[str] = None,
+    startTime: Optional[datetime] = datetime.now() - timedelta(days=30),
+    endTime: Optional[datetime] = datetime.now(),
 ):
-    global prisma_client
+    global prisma_client, llm_router
     if prisma_client is None:
         raise ProxyException(
             message="Prisma Client is not initialized",
@@ -7332,24 +7335,50 @@ async def model_metrics(
             param="None",
             code=status.HTTP_500_INTERNAL_SERVER_ERROR,
         )
+    if _selected_model_group and llm_router is not None:
+        _model_list = llm_router.get_model_list()
+        _relevant_api_bases = []
+        for model in _model_list:
+            if model["model_name"] == _selected_model_group:
+                _litellm_params = model["litellm_params"]
+                _api_base = _litellm_params.get("api_base", "")
+                _relevant_api_bases.append(_api_base)
+                _relevant_api_bases.append(_api_base + "/openai/")
 
-    sql_query = """
-        SELECT
-            CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base,
-            COUNT(*) AS num_requests,
-            AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) AS avg_latency_seconds
-        FROM
-            "LiteLLM_SpendLogs"
-        WHERE
-            "startTime" >= NOW() - INTERVAL '10000 hours'
-        GROUP BY
-            CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END
-        ORDER BY
-            num_requests DESC
-        LIMIT 50;
-    """
+        sql_query = """
+            SELECT
+                CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base,
+                COUNT(*) AS num_requests,
+                AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) AS avg_latency_seconds
+            FROM "LiteLLM_SpendLogs"
+            WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp
+            AND api_base = ANY($3)
+            GROUP BY CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END
+            ORDER BY num_requests DESC
+            LIMIT 50;
+        """
 
-    db_response = await prisma_client.db.query_raw(query=sql_query)
+        db_response = await prisma_client.db.query_raw(
+            sql_query, startTime, endTime, _relevant_api_bases
+        )
+    else:
+
+        sql_query = """
+            SELECT
+                CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base,
+                COUNT(*) AS num_requests,
+                AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) AS avg_latency_seconds
+            FROM
+                "LiteLLM_SpendLogs"
+            WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp
+            GROUP BY
+                CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END
+            ORDER BY
+                num_requests DESC
+            LIMIT 50;
+        """
+
+        db_response = await prisma_client.db.query_raw(sql_query, startTime, endTime)
     response: List[dict] = []
     if response is not None:
         # loop through all models

ui/litellm-dashboard/src/components/networking.tsx

@@ -403,13 +403,17 @@ export const modelInfoCall = async (
 export const modelMetricsCall = async (
   accessToken: String,
   userID: String,
-  userRole: String
+  userRole: String, 
+  modelGroup: String | null,
 ) => {
   /**
    * Get all models on proxy
    */
   try {
     let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics` : `/model/metrics`;
+    if (modelGroup) {
+      url = `${url}?_selected_model_group=${modelGroup}`
+    }
     // message.info("Requesting model data");
     const response = await fetch(url, {
       method: "GET",

ui/litellm-dashboard/src/components/usage.tsx

@@ -3,7 +3,7 @@ import { BarChart, BarList, Card, Title, Table, TableHead, TableHeaderCell, Tabl
 import React, { useState, useEffect } from "react";
 
 import ViewUserSpend from "./view_user_spend";
-import { Grid, Col, Text, LineChart, TabPanel, TabPanels, TabGroup, TabList, Tab } from "@tremor/react";
+import { Grid, Col, Text, LineChart, TabPanel, TabPanels, TabGroup, TabList, Tab, Select, SelectItem } from "@tremor/react";
 import {
   userSpendLogsCall,
   keyInfoCall,
@@ -13,6 +13,8 @@ import {
   teamSpendLogsCall,
   tagsSpendLogsCall,
   modelMetricsCall,
+  modelAvailableCall,
+  modelInfoCall,
 } from "./networking";
 import { start } from "repl";
 
@@ -146,6 +148,8 @@ const UsagePage: React.FC<UsagePageProps> = ({
   const [totalSpendPerTeam, setTotalSpendPerTeam] = useState<any[]>([]);
   const [modelMetrics, setModelMetrics] = useState<any[]>([]);
   const [modelLatencyMetrics, setModelLatencyMetrics] = useState<any[]>([]);
+  const [modelGroups, setModelGroups] = useState<any[]>([]);
+  const [selectedModelGroup, setSelectedModelGroup] = useState<string | null>(null);
 
   const firstDay = new Date(
     currentDate.getFullYear(),
@@ -227,7 +231,25 @@ const UsagePage: React.FC<UsagePageProps> = ({
             const top_tags = await tagsSpendLogsCall(accessToken);
             setTopTagsData(top_tags.top_10_tags);
 
-            
+            // get model groups 
+            const _model_groups = await modelInfoCall(accessToken, userID, userRole);
+            let model_groups = _model_groups.data;
+            console.log("model groups in model dashboard", model_groups);
+
+            let available_model_groups = [];
+            // loop through each model in model_group, access litellm_params and only inlclude the model if model["litellm_params"]["model"] startswith "azure/"
+            for (let i = 0; i < model_groups.length; i++) {
+              let model = model_groups[i];
+              console.log("model check", model);
+              let model_group = model["litellm_params"]["model"];
+              console.log("model group", model_group);
+              if (model_group.startsWith("azure/")) {
+                available_model_groups.push(model["model_name"]);
+              }
+            }
+            setModelGroups(available_model_groups);
+
+
           } else if (userRole == "App Owner") {
             await userSpendLogsCall(
               accessToken,
@@ -268,7 +290,8 @@ const UsagePage: React.FC<UsagePageProps> = ({
           const modelMetricsResponse = await modelMetricsCall(
             accessToken,
             userID,
-            userRole
+            userRole,
+            null
           );
   
           console.log("Model metrics response:", modelMetricsResponse);
@@ -288,6 +311,31 @@ const UsagePage: React.FC<UsagePageProps> = ({
     }
   }, [accessToken, token, userRole, userID, startTime, endTime]);
 
+
+  const updateModelMetrics = async (modelGroup: string | null) => {
+    console.log("Updating model metrics for group:", modelGroup);
+    if (!accessToken || !userID || !userRole) {
+      return
+    }
+    setSelectedModelGroup(modelGroup);  // If you want to store the selected model group in state
+
+  
+    try {
+      const modelMetricsResponse = await modelMetricsCall(accessToken, userID, userRole, modelGroup);
+      console.log("Model metrics response:", modelMetricsResponse);
+  
+      // Assuming modelMetricsResponse now contains the metric data for the specified model group
+      const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_seconds - a.avg_latency_seconds);
+      console.log("Sorted by latency:", sortedByLatency);
+  
+      setModelMetrics(modelMetricsResponse);
+      setModelLatencyMetrics(sortedByLatency);
+    } catch (error) {
+      console.error("Failed to fetch model metrics", error);
+    }
+  }
+  
+
   return (
     <div style={{ width: "100%" }} className="p-8">
       <ViewUserSpend
@@ -404,7 +452,7 @@ const UsagePage: React.FC<UsagePageProps> = ({
             </TabPanel>
             <TabPanel>
             <Grid numItems={2} className="gap-2 h-[75vh] w-full mb-4">
-              <Col numColSpan={2}>
+            <Col numColSpan={2}>
 
               <Card>
               <Title>Spend Per Tag - Last 30 Days</Title>
@@ -445,6 +493,31 @@ const UsagePage: React.FC<UsagePageProps> = ({
             </TabPanel>
             
             <TabPanel>
+              <Title>Filter By Model Group</Title>
+              <p style={{fontSize: '0.85rem', color: '#808080'}}>View how requests were load balanced within a model group</p>
+              <p style={{fontSize: '0.85rem', color: '#808080', fontStyle: 'italic'}}>(Beta feature) only supported for Azure Model Groups</p>
+
+
+            <Select
+              className="mb-4 mt-2"
+              defaultValue="all"
+            >
+              <SelectItem 
+                  value={"all"}
+                  onClick={() => updateModelMetrics(null)}
+                >
+                  All Model Groups
+                </SelectItem>
+              {modelGroups.map((group, idx) => (
+                <SelectItem 
+                  key={idx} 
+                  value={group}
+                  onClick={() => updateModelMetrics(group)}
+                >
+                  {group}
+                </SelectItem>
+              ))}
+            </Select>
             <Card>
           <Title>Number Requests per Model</Title>
               <BarChart