From bc95d3f44b4b5d874e9f7ac369fe14e6095e6078 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 12 Feb 2024 15:00:16 -0800 Subject: [PATCH 1/4] (ui) show models page --- ui/litellm-dashboard/src/app/page.tsx | 11 +++- .../src/components/leftnav.tsx | 5 +- .../src/components/model_dashboard.tsx | 63 +++++++++++++++++++ .../src/components/networking.tsx | 35 +++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/model_dashboard.tsx diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 80a3b6940ab..02e72b57971 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -3,6 +3,7 @@ import React, { Suspense, useEffect, useState } from "react"; import { useSearchParams } from "next/navigation"; import Navbar from "../components/navbar"; import UserDashboard from "../components/user_dashboard"; +import ModelDashboard from "@/components/model_dashboard"; import Sidebar from "../components/leftnav"; import Usage from "../components/usage"; import { jwtDecode } from "jwt-decode"; @@ -80,7 +81,15 @@ const CreateKeyPage = () => { userEmail={userEmail} setUserEmail={setUserEmail} /> - ) : ( + ) : page == "models" ? ( + + ) + : ( = ({ setPage }) => { setPage("api-keys")}> API Keys - setPage("usage")}> + setPage("models")}> + Models + + setPage("usage")}> Usage diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx new file mode 100644 index 00000000000..6067fc68610 --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -0,0 +1,63 @@ +import React, { useState, useEffect } from "react"; +import { Card, Title, Table, TableHead, TableRow, TableCell, TableBody } from "@tremor/react"; +import { modelInfoCall } from "./networking"; + +interface ModelDashboardProps { + accessToken: string | null; + token: string | null; + userRole: string | null; + userID: string | null; +} + +const ModelDashboard: React.FC = ({ + accessToken, + token, + userRole, + userID, +}) => { + const [modelData, setModelData] = useState({ data: [] }); + + useEffect(() => { + const fetchData = async () => { + try { + // Replace with your actual API call for model data + const modelDataResponse = await modelInfoCall(accessToken, token, userRole, userID); + + setModelData(modelDataResponse); + } catch (error) { + console.error("There was an error fetching the model data", error); + } + }; + + if (accessToken && token && userRole && userID) { + fetchData(); + } + }, [accessToken, token, userRole, userID]); + + return ( +
+ + Models Page + + + + Model Name + Model Info + + + + {modelData.data.map((model: any) => ( + + {model.model_name} + {/* {model.model_info} */} + {/* Add other TableCell for Model Info if needed */} + + ))} + +
+ +
+ ); +}; + +export default ModelDashboard; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 4521ca44661..7905ed7dbcd 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -137,6 +137,41 @@ export const userInfoCall = async ( } }; + + +export const modelInfoCall = async ( + accessToken: String, + userID: String, + userRole: String +) => { + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/model/info` : `/model/info`; + + message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + export const keySpendLogsCall = async (accessToken: String, token: String) => { try { const url = proxyBaseUrl ? `${proxyBaseUrl}/spend/logs` : `/spend/logs`; From 033c8d010c2dffbf384ccaf6d5cfbdeebdbf7e00 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 12 Feb 2024 15:42:42 -0800 Subject: [PATCH 2/4] (fix) allow ui to show /model/info --- litellm/proxy/proxy_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4927f3db4ca..2baf470ba12 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -691,6 +691,7 @@ async def user_api_key_auth( "/key", "/spend", "/user", + "/model/info", ] # check if the current route startswith any of the allowed routes if ( From cd1416e39c34c44f7905684c2fec4f3a9dc1fb39 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 12 Feb 2024 16:13:36 -0800 Subject: [PATCH 3/4] (feat) model/info have fallback check --- litellm/proxy/proxy_server.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 2baf470ba12..3faadfe5122 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3954,7 +3954,6 @@ async def add_new_model(model_params: ModelParams): ) -#### [BETA] - This is a beta endpoint, format might change based on user feedback https://github.com/BerriAI/litellm/issues/933. If you need a stable endpoint use /model/info @router.get( "/model/info", description="Provides more info about each model in /models, including config.yaml descriptions (except api key and api base)", @@ -3987,6 +3986,14 @@ async def model_info_v1( # read litellm model_prices_and_context_window.json to get the following: # input_cost_per_token, output_cost_per_token, max_tokens litellm_model_info = get_litellm_model_info(model=model) + if litellm_model_info == {}: + # use litellm_param model_name to get model_info + litellm_params = model.get("litellm_params", {}) + litellm_model = litellm_params.get("model", None) + try: + litellm_model_info = litellm.get_model_info(model=litellm_model) + except: + litellm_model_info = {} for k, v in litellm_model_info.items(): if k not in model_info: model_info[k] = v From 0a48177fc8b7f445cafe3b2a4b77f9464fd6430b Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 12 Feb 2024 16:15:56 -0800 Subject: [PATCH 4/4] (feat) show model info on dashboard --- .../src/components/model_dashboard.tsx | 77 ++++++++++++++++--- 1 file changed, 68 insertions(+), 9 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 6067fc68610..a454399f09b 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -1,9 +1,9 @@ import React, { useState, useEffect } from "react"; -import { Card, Title, Table, TableHead, TableRow, TableCell, TableBody } from "@tremor/react"; +import { Card, Title, Subtitle, Table, TableHead, TableRow, TableCell, TableBody, Metric, Grid } from "@tremor/react"; import { modelInfoCall } from "./networking"; interface ModelDashboardProps { - accessToken: string | null; + accessToken: string; token: string | null; userRole: string | null; userID: string | null; @@ -21,8 +21,8 @@ const ModelDashboard: React.FC = ({ const fetchData = async () => { try { // Replace with your actual API call for model data - const modelDataResponse = await modelInfoCall(accessToken, token, userRole, userID); - + const modelDataResponse = await modelInfoCall(accessToken, userID, userRole); + console.log("Model data response:", modelDataResponse.data); setModelData(modelDataResponse); } catch (error) { console.error("There was an error fetching the model data", error); @@ -34,28 +34,87 @@ const ModelDashboard: React.FC = ({ } }, [accessToken, token, userRole, userID]); + if (!modelData) { + return
Loading...
; + } + + // loop through model data and edit each row + for (let i = 0; i < modelData.data.length; i++) { + let curr_model = modelData.data[i]; + let litellm_model_name = curr_model?.litellm_params?.model; + + let model_info = curr_model?.model_info; + + let defaultProvider = "openai"; + let provider = ""; + let input_cost = "Undefined" + let output_cost = "Undefined" + let max_tokens = "Undefined" + + // Check if litellm_model_name is null or undefined + if (litellm_model_name) { + // Split litellm_model_name based on "/" + let splitModel = litellm_model_name.split("/"); + + // Get the first element in the split + let firstElement = splitModel[0]; + + // If there is only one element, default provider to openai + provider = splitModel.length === 1 ? defaultProvider : firstElement; + + console.log("Provider:", provider); + } else { + // litellm_model_name is null or undefined, default provider to openai + provider = defaultProvider; + console.log("Provider:", provider); + } + + if (model_info) { + input_cost = model_info?.input_cost_per_token; + output_cost = model_info?.output_cost_per_token; + max_tokens = model_info?.max_tokens; + + } + modelData.data[i].provider = provider + modelData.data[i].input_cost = input_cost + modelData.data[i].output_cost = output_cost + modelData.data[i].max_tokens = max_tokens + + } + + return (
+ - Models Page + Available Models Model Name - Model Info + Provider + Input Price per token ($) + Output Price per token ($) + Max Tokens {modelData.data.map((model: any) => ( - {model.model_name} - {/* {model.model_info} */} - {/* Add other TableCell for Model Info if needed */} + +

{model.model_name}

 + {model.provider} + {model.input_cost} + {model.output_cost} + {model.max_tokens} + + ))}
+
); };