From 12343650e4ffe15ae0aa47f3130c9aac94dd2a34 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 4 Jul 2024 13:13:42 -0700 Subject: [PATCH 1/7] feat - allow looking up model_id in model info --- .../common_utils/openai_endpoint_utils.py | 21 +++++++++++ litellm/proxy/proxy_server.py | 35 ++++++++++++++++--- 2 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 litellm/proxy/common_utils/openai_endpoint_utils.py diff --git a/litellm/proxy/common_utils/openai_endpoint_utils.py b/litellm/proxy/common_utils/openai_endpoint_utils.py new file mode 100644 index 00000000000..4d3f4220b90 --- /dev/null +++ b/litellm/proxy/common_utils/openai_endpoint_utils.py @@ -0,0 +1,21 @@ +""" +Contains utils used by OpenAI compatible endpoints +""" + + +def remove_sensitive_info_from_deployment(deployment_dict: dict) -> dict: + """ + Removes sensitive information from a deployment dictionary. + + Args: + deployment_dict (dict): The deployment dictionary to remove sensitive information from. + + Returns: + dict: The modified deployment dictionary with sensitive information removed. + """ + deployment_dict["litellm_params"].pop("api_key", None) + deployment_dict["litellm_params"].pop("vertex_credentials", None) + deployment_dict["litellm_params"].pop("aws_access_key_id", None) + deployment_dict["litellm_params"].pop("aws_secret_access_key", None) + + return deployment_dict diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9f745bb54de..7794f2a4cd1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -143,6 +143,9 @@ from litellm.proxy.caching_routes import router as caching_router from litellm.proxy.common_utils.debug_utils import router as debugging_endpoints_router from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.proxy.common_utils.init_callbacks import initialize_callbacks_on_proxy +from litellm.proxy.common_utils.openai_endpoint_utils import ( + remove_sensitive_info_from_deployment, +) from litellm.proxy.guardrails.init_guardrails import initialize_guardrails from litellm.proxy.health_check import perform_health_check from litellm.proxy.health_endpoints._health_endpoints import router as health_router @@ -6657,14 +6660,39 @@ async def model_metrics_exceptions( ) async def model_info_v1( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + litellm_model_id: Optional[str] = None, ): - global llm_model_list, general_settings, user_config_file_path, proxy_config + global llm_model_list, general_settings, user_config_file_path, proxy_config, llm_router if llm_model_list is None: raise HTTPException( status_code=500, detail={"error": "LLM Model List not loaded in"} ) + if llm_router is None: + raise HTTPException( + status_code=500, + detail={ + "error": "LLM Router is not loaded in. Make sure you passed models in your config.yaml or on the LiteLLM Admin UI." + }, + ) + + if litellm_model_id is not None: + # user is trying to get specific model from litellm router + deployment_info = llm_router.get_deployment(model_id=litellm_model_id) + if deployment_info is None: + raise HTTPException( + status_code=404, + detail={ + "error": f"Model id = {litellm_model_id} not found on litellm proxy" + }, + ) + _deployment_info_dict = deployment_info.model_dump() + _deployment_info_dict = remove_sensitive_info_from_deployment( + deployment_dict=_deployment_info_dict + ) + return {"data": _deployment_info_dict} + all_models: List[dict] = [] ## CHECK IF MODEL RESTRICTIONS ARE SET AT KEY/TEAM LEVEL ## if llm_model_list is None: @@ -6726,10 +6754,7 @@ async def model_info_v1( model_info[k] = v model["model_info"] = model_info # don't return the llm credentials - model["litellm_params"].pop("api_key", None) - model["litellm_params"].pop("vertex_credentials", None) - model["litellm_params"].pop("aws_access_key_id", None) - model["litellm_params"].pop("aws_secret_access_key", None) + model = remove_sensitive_info_from_deployment(deployment_dict=model) verbose_proxy_logger.debug("all_models: %s", all_models) return {"data": all_models} From a457aec76125417270630a8f849700518cdabf21 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 4 Jul 2024 13:21:02 -0700 Subject: [PATCH 2/7] add doc string for v1/model/info --- litellm/proxy/proxy_server.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7794f2a4cd1..ad6c83b6f14 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6648,13 +6648,11 @@ async def model_metrics_exceptions( @router.get( "/model/info", - description="Provides more info about each model in /models, including config.yaml descriptions (except api key and api base)", tags=["model management"], dependencies=[Depends(user_api_key_auth)], ) @router.get( "/v1/model/info", - description="Provides more info about each model in /models, including config.yaml descriptions (except api key and api base)", tags=["model management"], dependencies=[Depends(user_api_key_auth)], ) @@ -6662,6 +6660,38 @@ async def model_info_v1( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), litellm_model_id: Optional[str] = None, ): + """ + Provides more info about each model in /models, including config.yaml descriptions (except api key and api base) + + Parameters: + litellm_model_id: Optional[str] = None (this is the value of `x-litellm-model-id` returned in response headers) + + - When litellm_model_id is passed, it will return the info for that specific model + - When litellm_model_id is not passed, it will return the info for all models + + Returns: + Returns a dictionary containing information about each model. + + Example Response: + ```json + { + "data": [ + { + "model_name": "fake-openai-endpoint", + "litellm_params": { + "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", + "model": "openai/fake" + }, + "model_info": { + "id": "112f74fab24a7a5245d2ced3536dd8f5f9192c57ee6e332af0f0512e08bed5af", + "db_model": false + } + } + ] + } + + ``` + """ global llm_model_list, general_settings, user_config_file_path, proxy_config, llm_router if llm_model_list is None: From fe15df26f2cd2ffa856bfc37864b8bc743d74592 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 4 Jul 2024 15:40:08 -0700 Subject: [PATCH 3/7] fix invite link flow --- .../src/components/user_dashboard.tsx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/user_dashboard.tsx b/ui/litellm-dashboard/src/components/user_dashboard.tsx index ec66bcea0fa..2d2fb0dc1e4 100644 --- a/ui/litellm-dashboard/src/components/user_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/user_dashboard.tsx @@ -12,6 +12,7 @@ import ViewKeyTable from "./view_key_table"; import ViewUserSpend from "./view_user_spend"; import ViewUserTeam from "./view_user_team"; import DashboardTeam from "./dashboard_default_team"; +import Onboarding from "../app/onboarding/page"; import { useSearchParams, useRouter } from "next/navigation"; import { jwtDecode } from "jwt-decode"; import { Typography } from "antd"; @@ -76,6 +77,8 @@ const UserDashboard: React.FC = ({ const token = getCookie('token'); + const invitation_id = searchParams.get("invitation_id"); + const [accessToken, setAccessToken] = useState(null); const [teamSpend, setTeamSpend] = useState(null); const [userModels, setUserModels] = useState([]); @@ -257,7 +260,13 @@ const UserDashboard: React.FC = ({ }, [selectedTeam]); if (userID == null || token == null) { - // Now you can construct the full URL + // user is not logged in as yet + if (invitation_id != null) { + return ( + + + ) + } const url = proxyBaseUrl ? `${proxyBaseUrl}/sso/key/generate` : `/sso/key/generate`; From c3bbc4a69235f68b27f84b21969e282392835888 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 4 Jul 2024 15:40:19 -0700 Subject: [PATCH 4/7] fix invite link flow --- ui/litellm-dashboard/src/components/onboarding_link.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/onboarding_link.tsx b/ui/litellm-dashboard/src/components/onboarding_link.tsx index 09a635a1440..3889e1ef945 100644 --- a/ui/litellm-dashboard/src/components/onboarding_link.tsx +++ b/ui/litellm-dashboard/src/components/onboarding_link.tsx @@ -67,13 +67,13 @@ const OnboardingModal: React.FC = ({
Invitation Link - {baseUrl}/ui/onboarding?id={invitationLinkData?.id} + {baseUrl}/ui?invitation_id={invitationLinkData?.id}
message.success("Copied!")} > From fdcdee8775d170c18c212f6ce1bdaa58fcbcc25e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 4 Jul 2024 15:40:49 -0700 Subject: [PATCH 5/7] fix invite id flow --- .../src/app/onboarding/page.tsx | 9 +++-- ui/litellm-dashboard/src/app/page.tsx | 34 +++++++++++++++---- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/app/onboarding/page.tsx b/ui/litellm-dashboard/src/app/onboarding/page.tsx index a31afb0f9e5..373944a98d3 100644 --- a/ui/litellm-dashboard/src/app/onboarding/page.tsx +++ b/ui/litellm-dashboard/src/app/onboarding/page.tsx @@ -33,7 +33,7 @@ export default function Onboarding() { const [form] = Form.useForm(); const searchParams = useSearchParams(); const token = getCookie('token'); - const inviteID = searchParams.get("id"); + const inviteID = searchParams.get("invitation_id"); const [accessToken, setAccessToken] = useState(null); const [defaultUserEmail, setDefaultUserEmail] = useState(""); const [userEmail, setUserEmail] = useState(""); @@ -92,7 +92,10 @@ export default function Onboarding() { ).then((data) => { let litellm_dashboard_ui = "/ui/"; const user_id = data.data?.user_id || data.user_id; - litellm_dashboard_ui += "?userID=" + user_id + "&token=" + jwtToken; + litellm_dashboard_ui += "?userID=" + user_id; + + // set cookie "token" to jwtToken + document.cookie = "token=" + jwtToken; console.log("redirecting to:", litellm_dashboard_ui); window.location.href = litellm_dashboard_ui; @@ -101,7 +104,7 @@ export default function Onboarding() { // redirect to login page }; return ( -
+
🚅 LiteLLM Sign up diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 41f0e8529b3..1aaf8cbc3d8 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -77,6 +77,7 @@ const CreateKeyPage = () => { const searchParams = useSearchParams(); const [modelData, setModelData] = useState({ data: [] }); const userID = searchParams.get("userID"); + const invitation_id = searchParams.get("invitation_id"); const token = getCookie('token'); const [page, setPage] = useState("api-keys"); @@ -128,7 +129,23 @@ const CreateKeyPage = () => { return ( Loading...
}> -
+ { + invitation_id ? ( + + ) : ( +
{ />
- +
{page == "api-keys" ? ( @@ -250,7 +267,10 @@ const CreateKeyPage = () => { /> )}
-
+
+ ) + } + ); }; From 7164e728ed2e5c72b9be3c034d0b60dade85449e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 4 Jul 2024 15:54:11 -0700 Subject: [PATCH 6/7] fix invite link sign up --- .../src/components/user_dashboard.tsx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/user_dashboard.tsx b/ui/litellm-dashboard/src/components/user_dashboard.tsx index 2d2fb0dc1e4..fdddcaad62e 100644 --- a/ui/litellm-dashboard/src/components/user_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/user_dashboard.tsx @@ -259,14 +259,15 @@ const UserDashboard: React.FC = ({ } }, [selectedTeam]); + + if (invitation_id != null) { + return ( + + ) + } + if (userID == null || token == null) { // user is not logged in as yet - if (invitation_id != null) { - return ( - - - ) - } const url = proxyBaseUrl ? `${proxyBaseUrl}/sso/key/generate` : `/sso/key/generate`; From c1c2f9faa435cbe59238035352881ee7e7e10e5e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 4 Jul 2024 15:55:04 -0700 Subject: [PATCH 7/7] fix backend invite link sign up --- litellm/proxy/proxy_server.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ad6c83b6f14..f388db4a712 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7704,22 +7704,12 @@ async def claim_onboarding_link(data: InvitationClaim): ) #### CHECK IF CLAIMED - ##### if claimed - check if within valid session (within 10 minutes of being claimed) + ##### if claimed - accept ##### if unclaimed - reject - current_time = litellm.utils.get_utc_datetime() - - if invite_obj.is_accepted == True: - time_difference = current_time - invite_obj.updated_at - - # Check if the difference is within 10 minutes - if time_difference > timedelta(minutes=10): - raise HTTPException( - status_code=401, - detail={ - "error": "The invitation link has already been claimed. Please ask your admin for a new invite link." - }, - ) + if invite_obj.is_accepted is True: + # this is a valid invite that was accepted + pass else: raise HTTPException( status_code=401,