From 9b8befe0523e0e1603e6fa55e216df81efa1697d Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 19 Feb 2024 09:25:32 -0800 Subject: [PATCH 1/7] (feat) /model/info show models user has access to --- litellm/proxy/proxy_server.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 295c8094160..45219d30709 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4327,11 +4327,29 @@ async def model_info_v1( # Load existing config config = await proxy_config.get_config() + all_models = config.get("model_list", []) + if user_model is not None: + # if user does not use a config.yaml, https://github.com/BerriAI/litellm/issues/2061 + all_models += [user_model] + + # check all models user has access to in user_api_key_dict + user_models = [] if len(user_api_key_dict.models) > 0: model_names = user_api_key_dict.models - all_models = [m for m in config["model_list"] if m in model_names] - else: - all_models = config["model_list"] + user_models = [m for m in config["model_list"] if m in model_names] + + # for all models check if the user has access, and mark it as "user_access": `True` or `False` + for model in all_models: + model_name = model.get("model_name", None) + if model_name is not None: + user_has_access = model_name in user_models + if ( + user_models == [] + ): # if user_api_key_dict.models == [], user has access to all models + user_has_access = True + model["user_access"] = user_has_access + + # fill in model info based on config.yaml and litellm model_prices_and_context_window.json for model in all_models: # provided model_info in config.yaml model_info = model.get("model_info", {}) From 75db87fb71ae158559c1025189ef88123aaf48ac Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 19 Feb 2024 13:15:46 -0800 Subject: [PATCH 2/7] (ui) show models a user has access to --- .../src/components/model_dashboard.tsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 707d03b851d..1cc2d98a83a 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -1,6 +1,7 @@ import React, { useState, useEffect } from "react"; import { Card, Title, Subtitle, Table, TableHead, TableRow, TableCell, TableBody, Metric, Grid } from "@tremor/react"; import { modelInfoCall } from "./networking"; +import { Badge, BadgeDelta } from '@tremor/react'; interface ModelDashboardProps { accessToken: string | null; @@ -65,11 +66,9 @@ const ModelDashboard: React.FC = ({ // If there is only one element, default provider to openai provider = splitModel.length === 1 ? defaultProvider : firstElement; - console.log("Provider:", provider); } else { // litellm_model_name is null or undefined, default provider to openai provider = defaultProvider; - console.log("Provider:", provider); } if (model_info) { @@ -83,8 +82,9 @@ const ModelDashboard: React.FC = ({ modelData.data[i].output_cost = output_cost modelData.data[i].max_tokens = max_tokens - } + console.log(modelData.data[i]); + } return (
@@ -95,6 +95,7 @@ const ModelDashboard: React.FC = ({ Model Name Provider + Access Input Price per token ($) Output Price per token ($) Max Tokens @@ -106,6 +107,11 @@ const ModelDashboard: React.FC = ({ {model.model_name} {model.provider} + + + {model.user_access ? Yes : Request Access} + + {model.input_cost} {model.output_cost} {model.max_tokens} From c523e1d55cc9d8b821cadcb89cc25cffa271eaa2 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 19 Feb 2024 13:17:24 -0800 Subject: [PATCH 3/7] (fix) server show models user has access to --- litellm/proxy/proxy_server.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 45219d30709..f82a2fc96d8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4335,8 +4335,7 @@ async def model_info_v1( # check all models user has access to in user_api_key_dict user_models = [] if len(user_api_key_dict.models) > 0: - model_names = user_api_key_dict.models - user_models = [m for m in config["model_list"] if m in model_names] + user_models = user_api_key_dict.models # for all models check if the user has access, and mark it as "user_access": `True` or `False` for model in all_models: @@ -4978,8 +4977,25 @@ async def auth_callback(request: Request): if user_id is None: user_id = getattr(result, "first_name", "") + getattr(result, "last_name", "") + # get user_info from litellm DB + user_info = None + if prisma_client is not None: + user_info = await prisma_client.get_data(user_id=user_id, table_name="user") + if user_info is not None: + user_id_models = getattr(user_info, "models", []) + response = await generate_key_helper_fn( - **{"duration": "1hr", "key_max_budget": 0.01, "models": [], "aliases": {}, "config": {}, "spend": 0, "user_id": user_id, "team_id": "litellm-dashboard", "user_email": user_email} # type: ignore + **{ + "duration": "1hr", + "key_max_budget": 0.01, + "models": user_id_models, + "aliases": {}, + "config": {}, + "spend": 0, + "user_id": user_id, + "team_id": "litellm-dashboard", + "user_email": user_email, + } # type: ignore ) key = response["token"] # type: ignore user_id = response["user_id"] # type: ignore From 8c920f9533c87fade11ee5ce857990bdbe156240 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 19 Feb 2024 13:23:15 -0800 Subject: [PATCH 4/7] (fix) use /v2/model/info for UI --- ui/litellm-dashboard/src/components/networking.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 5874ae2a971..3a223504112 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -145,7 +145,7 @@ export const modelInfoCall = async ( userRole: String ) => { try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/model/info` : `/model/info`; + let url = proxyBaseUrl ? `${proxyBaseUrl}/v2/model/info` : `/v2/model/info`; message.info("Requesting model data"); const response = await fetch(url, { From e5cb0bbaf609cfe7fd6f20265dae381f00fae310 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 19 Feb 2024 13:24:22 -0800 Subject: [PATCH 5/7] (feat) use /v2/model/info --- litellm/proxy/proxy_server.py | 76 +++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f82a2fc96d8..9fee7d4a35d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4307,6 +4307,82 @@ async def add_new_model(model_params: ModelParams): ) +@router.get( + "/v2/model/info", + description="v2 - returns all the models set on the config.yaml, shows 'user_access' = True if the user has access to the model. Provides more info about each model in /models, including config.yaml descriptions (except api key and api base)", + tags=["model management"], + dependencies=[Depends(user_api_key_auth)], +) +async def model_info_v2( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + global llm_model_list, general_settings, user_config_file_path, proxy_config + + # Load existing config + config = await proxy_config.get_config() + + all_models = config.get("model_list", []) + if user_model is not None: + # if user does not use a config.yaml, https://github.com/BerriAI/litellm/issues/2061 + all_models += [user_model] + + # check all models user has access to in user_api_key_dict + user_models = [] + if len(user_api_key_dict.models) > 0: + user_models = user_api_key_dict.models + + # for all models check if the user has access, and mark it as "user_access": `True` or `False` + for model in all_models: + model_name = model.get("model_name", None) + if model_name is not None: + user_has_access = model_name in user_models + if ( + user_models == [] + ): # if user_api_key_dict.models == [], user has access to all models + user_has_access = True + model["user_access"] = user_has_access + + # fill in model info based on config.yaml and litellm model_prices_and_context_window.json + for model in all_models: + # provided model_info in config.yaml + model_info = model.get("model_info", {}) + + # read litellm model_prices_and_context_window.json to get the following: + # input_cost_per_token, output_cost_per_token, max_tokens + litellm_model_info = get_litellm_model_info(model=model) + + # 2nd pass on the model, try seeing if we can find model in litellm model_cost map + if litellm_model_info == {}: + # use litellm_param model_name to get model_info + litellm_params = model.get("litellm_params", {}) + litellm_model = litellm_params.get("model", None) + try: + litellm_model_info = litellm.get_model_info(model=litellm_model) + except: + litellm_model_info = {} + # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map + if litellm_model_info == {}: + # use litellm_param model_name to get model_info + litellm_params = model.get("litellm_params", {}) + litellm_model = litellm_params.get("model", None) + split_model = litellm_model.split("/") + if len(split_model) > 0: + litellm_model = split_model[-1] + try: + litellm_model_info = litellm.get_model_info(model=litellm_model) + except: + litellm_model_info = {} + for k, v in litellm_model_info.items(): + if k not in model_info: + model_info[k] = v + model["model_info"] = model_info + # don't return the api key + model["litellm_params"].pop("api_key", None) + + verbose_proxy_logger.debug(f"all_models: {all_models}") + return {"data": all_models} + + @router.get( "/model/info", description="Provides more info about each model in /models, including config.yaml descriptions (except api key and api base)", From 17a6db7ed4da40e4f8734a7b858695a32ac28bc8 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 19 Feb 2024 13:27:12 -0800 Subject: [PATCH 6/7] (fix) model/info --- litellm/proxy/proxy_server.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9fee7d4a35d..7b4943dc5b0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4403,28 +4403,11 @@ async def model_info_v1( # Load existing config config = await proxy_config.get_config() - all_models = config.get("model_list", []) - if user_model is not None: - # if user does not use a config.yaml, https://github.com/BerriAI/litellm/issues/2061 - all_models += [user_model] - - # check all models user has access to in user_api_key_dict - user_models = [] if len(user_api_key_dict.models) > 0: - user_models = user_api_key_dict.models - - # for all models check if the user has access, and mark it as "user_access": `True` or `False` - for model in all_models: - model_name = model.get("model_name", None) - if model_name is not None: - user_has_access = model_name in user_models - if ( - user_models == [] - ): # if user_api_key_dict.models == [], user has access to all models - user_has_access = True - model["user_access"] = user_has_access - - # fill in model info based on config.yaml and litellm model_prices_and_context_window.json + model_names = user_api_key_dict.models + all_models = [m for m in config["model_list"] if m in model_names] + else: + all_models = config["model_list"] for model in all_models: # provided model_info in config.yaml model_info = model.get("model_info", {}) From 050114bb2900bda25559f74323c313469a627466 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 19 Feb 2024 13:29:19 -0800 Subject: [PATCH 7/7] (fix) use "/v2/model/info", --- litellm/proxy/proxy_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7b4943dc5b0..ef2b6cac7ca 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -727,6 +727,7 @@ async def user_api_key_auth( "/spend", "/user", "/model/info", + "/v2/model/info", ] # check if the current route startswith any of the allowed routes if (