From f69f3721358da8d180f329c61dba416ccce5b9ca Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 27 May 2024 14:28:05 -0700 Subject: [PATCH 01/12] feat(model_hub/page.tsx): public model hub page for users allow admin to expose a public model hub page for users to see available models w/ params --- .../src/app/model_hub/page.tsx | 23 ++++ ui/litellm-dashboard/src/app/page.tsx | 55 ++++---- .../src/components/model_hub.tsx | 130 +++++++++--------- .../src/components/networking.tsx | 13 +- 4 files changed, 120 insertions(+), 101 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/model_hub/page.tsx diff --git a/ui/litellm-dashboard/src/app/model_hub/page.tsx b/ui/litellm-dashboard/src/app/model_hub/page.tsx new file mode 100644 index 00000000000..a038a697470 --- /dev/null +++ b/ui/litellm-dashboard/src/app/model_hub/page.tsx @@ -0,0 +1,23 @@ +"use client"; +import React, { Suspense, useEffect, useState } from "react"; +import { useSearchParams } from "next/navigation"; +import { modelHubCall } from "@/components/networking"; +import ModelHub from "@/components/model_hub"; + +export default function PublicModelHub() { + const searchParams = useSearchParams(); + const key = searchParams.get("key"); + const [accessToken, setAccessToken] = useState(null); + + useEffect(() => { + if (!key) { + return; + } + setAccessToken(key); + }, [key]); + /** + * populate navbar + * + */ + return ; +} diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 382d72b8dc2..8d2b1fac2ba 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -18,6 +18,30 @@ import Usage from "../components/usage"; import { jwtDecode } from "jwt-decode"; import { Typography } from "antd"; +export function formatUserRole(userRole: string) { + if (!userRole) { + return "Undefined Role"; + } + console.log(`Received user role: ${userRole.toLowerCase()}`); + console.log(`Received user role length: ${userRole.toLowerCase().length}`); + switch (userRole.toLowerCase()) { + case "app_owner": + return "App Owner"; + case "demo_app_owner": + return "App Owner"; + case "app_admin": + return "Admin"; + case "proxy_admin": + return "Admin"; + case "proxy_admin_viewer": + return "Admin Viewer"; + case "app_user": + return "App User"; + default: + return "Unknown Role"; + } +} + const CreateKeyPage = () => { const { Title, Paragraph } = Typography; const [userRole, setUserRole] = useState(""); @@ -78,30 +102,6 @@ const CreateKeyPage = () => { } }, [token]); - function formatUserRole(userRole: string) { - if (!userRole) { - return "Undefined Role"; - } - console.log(`Received user role: ${userRole.toLowerCase()}`); - console.log(`Received user role length: ${userRole.toLowerCase().length}`); - switch (userRole.toLowerCase()) { - case "app_owner": - return "App Owner"; - case "demo_app_owner": - return "App Owner"; - case "app_admin": - return "Admin"; - case "proxy_admin": - return "Admin"; - case "proxy_admin_viewer": - return "Admin Viewer"; - case "app_user": - return "App User"; - default: - return "Unknown Role"; - } - } - return ( Loading...}>
@@ -194,8 +194,8 @@ const CreateKeyPage = () => { accessToken={accessToken} modelData={modelData} /> - ) : page == "model-hub" ? ( - { keys={keys} premiumUser={premiumUser} /> - ) - : ( + ) : ( = ({ - userID, - - userRole, - - token, - - accessToken, - - keys, - - premiumUser, -}) => { +const ModelHub: React.FC = ({ accessToken, publicPage }) => { const [modelHubData, setModelHubData] = useState(null); const [isModalVisible, setIsModalVisible] = useState(false); const [selectedModel, setSelectedModel] = useState(null); useEffect(() => { - if (!accessToken || !token || !userRole || !userID) { + if (!accessToken) { return; } const fetchData = async () => { try { - const _modelHubData = await modelHubCall(accessToken, userID, userRole); + const _modelHubData = await modelHubCall(accessToken); console.log("ModelHubData:", _modelHubData); @@ -78,7 +59,7 @@ const ModelHub: React.FC = ({ }; fetchData(); - }, [accessToken, token, userRole, userID]); + }, [accessToken]); const showModal = (model: ModelInfo) => { setSelectedModel(model); @@ -109,11 +90,13 @@ const ModelHub: React.FC = ({
Model Hub - + {publicPage == false && ( + + )}
@@ -129,14 +112,27 @@ const ModelHub: React.FC = ({ /> -
- - Mode: {model.mode} - Supports Function Calling: {model?.supports_function_calling == true ? "Yes" : "No"} - Supports Vision: {model?.supports_vision == true ? "Yes" : "No"} - Max Input Tokens: {model?.max_input_tokens ? model?.max_input_tokens : "N/A"} - Max Output Tokens: {model?.max_output_tokens ? model?.max_output_tokens : "N/A"} -
+
+ Mode: {model.mode} + + Supports Function Calling:{" "} + {model?.supports_function_calling == true ? "Yes" : "No"} + + + Supports Vision:{" "} + {model?.supports_vision == true ? "Yes" : "No"} + + + Max Input Tokens:{" "} + {model?.max_input_tokens ? model?.max_input_tokens : "N/A"} + + + Max Output Tokens:{" "} + {model?.max_output_tokens + ? model?.max_output_tokens + : "N/A"} + +
= ({ > {selectedModel && (
-

Model Information & Usage

- +

+ Model Information & Usage +

+ - - OpenAI Python SDK - Supported OpenAI Params - LlamaIndex - Langchain Py - - - - - {` + + OpenAI Python SDK + Supported OpenAI Params + LlamaIndex + Langchain Py + + + + + {` import openai client = openai.OpenAI( api_key="your_api_key", @@ -192,13 +194,13 @@ response = client.chat.completions.create( print(response) `} - - - - - {`${selectedModel.supported_openai_params?.map((param) => `${param}\n`).join('')}`} - - + + + + + {`${selectedModel.supported_openai_params?.map((param) => `${param}\n`).join("")}`} + + {` diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 1820836614a..49d0a7453fe 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -579,17 +579,14 @@ export const modelInfoCall = async ( } }; - -export const modelHubCall = async ( - accessToken: String, - userID: String, - userRole: String -) => { +export const modelHubCall = async (accessToken: String) => { /** * Get all models on proxy */ try { - let url = proxyBaseUrl ? `${proxyBaseUrl}/model_group/info` : `/model_group/info`; + let url = proxyBaseUrl + ? `${proxyBaseUrl}/model_group/info` + : `/model_group/info`; //message.info("Requesting model data"); const response = await fetch(url, { @@ -617,8 +614,6 @@ export const modelHubCall = async ( } }; - - export const modelMetricsCall = async ( accessToken: String, userID: String, From 4516daec90f63b24d2f4c287be6a95683ce5fc88 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 27 May 2024 17:45:45 -0700 Subject: [PATCH 02/12] feat(model_hub.tsx): enable admin to expose a public model hub --- litellm/proxy/_types.py | 9 + litellm/proxy/proxy_server.py | 8 +- .../src/app/model_hub/page.tsx | 4 +- ui/litellm-dashboard/src/app/page.tsx | 5 +- .../src/components/model_hub.tsx | 210 ++++++++++++------ .../src/components/networking.tsx | 32 +++ 6 files changed, 193 insertions(+), 75 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e85f116f73b..0893427b723 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -928,6 +928,10 @@ class ConfigGeneralSettings(LiteLLMBase): allowed_routes: Optional[List] = Field( None, description="Proxy API Endpoints you want users to be able to access" ) + enable_public_model_hub: bool = Field( + default=False, + description="Public model hub for users to see what models they have access to, supported openai params, etc.", + ) class ConfigYAML(LiteLLMBase): @@ -1154,3 +1158,8 @@ class WebhookEvent(CallInfo): class SpecialModelNames(enum.Enum): all_team_models = "all-team-models" all_proxy_models = "all-proxy-models" + + +class ConfigFieldInfo(LiteLLMBase): + field_name: str + field_value: Any diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1bdb5edba30..3bc7ccbc59b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11061,6 +11061,7 @@ async def update_config_general_settings( "/config/field/info", tags=["config.yaml"], dependencies=[Depends(user_api_key_auth)], + response_model=ConfigFieldInfo, ) async def get_config_general_settings( field_name: str, @@ -11107,10 +11108,9 @@ async def get_config_general_settings( general_settings = dict(db_general_settings.param_value) if field_name in general_settings: - return { - "field_name": field_name, - "field_value": general_settings[field_name], - } + return ConfigFieldInfo( + field_name=field_name, field_value=general_settings[field_name] + ) else: raise HTTPException( status_code=400, diff --git a/ui/litellm-dashboard/src/app/model_hub/page.tsx b/ui/litellm-dashboard/src/app/model_hub/page.tsx index a038a697470..cca9f877b7f 100644 --- a/ui/litellm-dashboard/src/app/model_hub/page.tsx +++ b/ui/litellm-dashboard/src/app/model_hub/page.tsx @@ -19,5 +19,7 @@ export default function PublicModelHub() { * populate navbar * */ - return ; + return ( + + ); } diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index 8d2b1fac2ba..96bfa803001 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -196,11 +196,8 @@ const CreateKeyPage = () => { /> ) : page == "model-hub" ? ( ) : ( diff --git a/ui/litellm-dashboard/src/components/model_hub.tsx b/ui/litellm-dashboard/src/components/model_hub.tsx index 14b3cfd1f7c..5a141d154d1 100644 --- a/ui/litellm-dashboard/src/components/model_hub.tsx +++ b/ui/litellm-dashboard/src/components/model_hub.tsx @@ -1,7 +1,8 @@ import React, { useEffect, useState } from "react"; +import { useRouter, usePathname, useSearchParams } from "next/navigation"; import { modelHubCall } from "./networking"; - +import { getConfigFieldSetting, updateConfigFieldSetting } from "./networking"; import { Card, Text, @@ -15,15 +16,15 @@ import { TabPanel, TabPanels, } from "@tremor/react"; - import { RightOutlined, CopyOutlined } from "@ant-design/icons"; -import { Modal, Tooltip } from "antd"; +import { Modal, Tooltip, message } from "antd"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; interface ModelHubProps { accessToken: string | null; publicPage: boolean; + premiumUser: boolean; } interface ModelInfo { @@ -36,10 +37,18 @@ interface ModelInfo { supported_openai_params?: string[]; } -const ModelHub: React.FC = ({ accessToken, publicPage }) => { +const ModelHub: React.FC = ({ + accessToken, + publicPage, + premiumUser, +}) => { + const [publicPageAllowed, setPublicPageAllowed] = useState(false); const [modelHubData, setModelHubData] = useState(null); const [isModalVisible, setIsModalVisible] = useState(false); + const [isPublicPageModalVisible, setIsPublicPageModalVisible] = + useState(false); const [selectedModel, setSelectedModel] = useState(null); + const router = useRouter(); useEffect(() => { if (!accessToken) { @@ -53,13 +62,24 @@ const ModelHub: React.FC = ({ accessToken, publicPage }) => { console.log("ModelHubData:", _modelHubData); setModelHubData(_modelHubData.data); + + getConfigFieldSetting(accessToken, "enable_public_model_hub") + .then((data) => { + console.log(`data: ${JSON.stringify(data)}`); + if (data.field_value == true) { + setPublicPageAllowed(true); + } + }) + .catch((error) => { + // do nothing + }); } catch (error) { console.error("There was an error fetching the model data", error); } }; fetchData(); - }, [accessToken]); + }, [accessToken, publicPage]); const showModal = (model: ModelInfo) => { setSelectedModel(model); @@ -67,15 +87,29 @@ const ModelHub: React.FC = ({ accessToken, publicPage }) => { setIsModalVisible(true); }; + const goToPublicModelPage = () => { + router.replace(`/model_hub?key=${accessToken}`); + }; + const handleMakePublicPage = async () => { + if (!accessToken) { + return; + } + updateConfigFieldSetting(accessToken, "enable_public_model_hub", true).then( + (data) => { + setIsPublicPageModalVisible(true); + } + ); + }; + const handleOk = () => { setIsModalVisible(false); - + setIsPublicPageModalVisible(false); setSelectedModel(null); }; const handleCancel = () => { setIsModalVisible(false); - + setIsPublicPageModalVisible(false); setSelectedModel(null); }; @@ -85,68 +119,112 @@ const ModelHub: React.FC = ({ accessToken, publicPage }) => { return (
-
-
+ {(publicPage && publicPageAllowed) || publicPage == false ? ( +
+
- - -
- {modelHubData && - modelHubData.map((model: ModelInfo) => ( - - 
-                  {model.model_group}
-                  
-                     copyToClipboard(model.model_group)}
-                      style={{ cursor: "pointer", marginRight: "10px" }}
-                    />
-                  
-
-
- Mode: {model.mode} - - Supports Function Calling:{" "} - {model?.supports_function_calling == true ? "Yes" : "No"} - - - Supports Vision:{" "} - {model?.supports_vision == true ? "Yes" : "No"} - - - Max Input Tokens:{" "} - {model?.max_input_tokens ? model?.max_input_tokens : "N/A"} - - - Max Output Tokens:{" "} - {model?.max_output_tokens - ? model?.max_output_tokens - : "N/A"} - -
- -
+ + ) + ) : ( +
+

Filter by key:

+ {`/ui/model_hub?key=`} +
+ )} +
+
+ {modelHubData && + modelHubData.map((model: ModelInfo) => ( + + 
+                    {model.model_group}
+                    
+                       copyToClipboard(model.model_group)}
+                        style={{ cursor: "pointer", marginRight: "10px" }}
+                      />
+                    
+
+
+ Mode: {model.mode} + + Supports Function Calling:{" "} + {model?.supports_function_calling == true ? "Yes" : "No"} + + + Supports Vision:{" "} + {model?.supports_vision == true ? "Yes" : "No"} + + + Max Input Tokens:{" "} + {model?.max_input_tokens + ? model?.max_input_tokens + : "N/A"} + + + Max Output Tokens:{" "} + {model?.max_output_tokens + ? model?.max_output_tokens + : "N/A"} + +
+ + + ))} +
+
+ ) : ( + + + Public Model Hub not enabled. + +

+ Ask your proxy admin to enable this on their Admin UI. +

+ + )} + + +
+
+ Shareable Link: + {`/ui/model_hub?key=`} +
+
+ +
+
+ { } }; +export const getConfigFieldSetting = async ( + accessToken: String, + fieldName: string +) => { + try { + let url = proxyBaseUrl + ? `${proxyBaseUrl}/config/field/info?field_name=${fieldName}` + : `/config/field/info?field_name=${fieldName}`; + + //message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to set callbacks:", error); + throw error; + } +}; + export const updateConfigFieldSetting = async ( accessToken: String, fieldName: string, From 17aa5aa214813c011bb950c99a8da3c1995ae7f8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 27 May 2024 18:12:17 -0700 Subject: [PATCH 03/12] fix(openai.py): fix deepinfra supported params --- litellm/llms/openai.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index 6197ec92243..05e6566ffa8 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -224,6 +224,7 @@ class DeepInfraConfig: def get_supported_openai_params(self): return [ + "stream", "frequency_penalty", "function_call", "functions", From 9b8c14360140d4b877d54977d577738876a1fc7c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 18:17:46 -0700 Subject: [PATCH 04/12] feat - rename end_user -> customer --- litellm/proxy/proxy_server.py | 83 +++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1bdb5edba30..b7985ce5ba9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7131,13 +7131,15 @@ async def global_predict_spend_logs(request: Request): #### INTERNAL USER MANAGEMENT #### @router.post( "/user/new", - tags=["user management"], + tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], response_model=NewUserResponse, ) async def new_user(data: NewUserRequest): """ - Use this to create a new user with a budget. This creates a new user and generates a new api key for the new user. The new api key is returned. + Use this to create a new INTERNAL user with a budget. + Internal Users can access LiteLLM Admin UI to make keys, request access to models. + This creates a new user and generates a new api key for the new user. The new api key is returned. Returns user id, budget + new key. @@ -7208,7 +7210,9 @@ async def new_user(data: NewUserRequest): @router.post( - "/user/auth", tags=["user management"], dependencies=[Depends(user_api_key_auth)] + "/user/auth", + tags=["Internal User management"], + dependencies=[Depends(user_api_key_auth)], ) async def user_auth(request: Request): """ @@ -7274,7 +7278,9 @@ async def user_auth(request: Request): @router.get( - "/user/info", tags=["user management"], dependencies=[Depends(user_api_key_auth)] + "/user/info", + tags=["Internal User management"], + dependencies=[Depends(user_api_key_auth)], ) async def user_info( user_id: Optional[str] = fastapi.Query( @@ -7446,7 +7452,9 @@ async def user_info( @router.post( - "/user/update", tags=["user management"], dependencies=[Depends(user_api_key_auth)] + "/user/update", + tags=["Internal User management"], + dependencies=[Depends(user_api_key_auth)], ) async def user_update(data: UpdateUserRequest): """ @@ -7540,7 +7548,7 @@ async def user_update(data: UpdateUserRequest): @router.post( "/user/request_model", - tags=["user management"], + tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], ) async def user_request_model(request: Request): @@ -7593,7 +7601,7 @@ async def user_request_model(request: Request): @router.get( "/user/get_requests", - tags=["user management"], + tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], ) async def user_get_requests(): @@ -7635,7 +7643,7 @@ async def user_get_requests(): @router.get( "/user/get_users", - tags=["user management"], + tags=["Internal User management"], dependencies=[Depends(user_api_key_auth)], ) async def get_users( @@ -7672,7 +7680,13 @@ async def get_users( @router.post( "/end_user/block", - tags=["End User Management"], + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], + include_in_schema=False, +) +@router.post( + "/customer/block", + tags=["Customer Management"], dependencies=[Depends(user_api_key_auth)], ) async def block_user(data: BlockUsers): @@ -7715,9 +7729,15 @@ async def block_user(data: BlockUsers): @router.post( "/end_user/unblock", - tags=["End User Management"], + tags=["Customer Management"], dependencies=[Depends(user_api_key_auth)], ) +@router.post( + "/customer/unblock", + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], + include_in_schema=False, +) async def unblock_user(data: BlockUsers): """ [BETA] Unblock calls with this user id @@ -7762,7 +7782,13 @@ async def unblock_user(data: BlockUsers): @router.post( "/end_user/new", - tags=["End User Management"], + tags=["Customer Management"], + include_in_schema=False, + dependencies=[Depends(user_api_key_auth)], +) +@router.post( + "/customer/new", + tags=["Customer Management"], dependencies=[Depends(user_api_key_auth)], ) async def new_end_user( @@ -7779,18 +7805,13 @@ async def new_end_user( Example curl: ``` - curl --location 'http://0.0.0.0:4000/end_user/new' \ + curl --location 'http://0.0.0.0:4000/customer/new' \ --header 'Authorization: Bearer sk-1234' \ --header 'Content-Type: application/json' \ --data '{ - "end_user_id" : "ishaan-jaff-3", <- specific customer - - "allowed_region": "eu" <- set region for models - - + - + "user_id" : "ishaan-jaff-3", + "allowed_region": "eu" "default_model": "azure/gpt-3.5-turbo-eu" <- all calls from this user, use this model? - }' # return end-user object @@ -7860,9 +7881,15 @@ async def new_end_user( return end_user_record +@router.get( + "/customer/info", + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], +) @router.get( "/end_user/info", - tags=["End User Management"], + tags=["Customer Management"], + include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) async def end_user_info( @@ -7885,9 +7912,15 @@ async def end_user_info( return user_info +@router.post( + "/customer/update", + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], +) @router.post( "/end_user/update", - tags=["End User Management"], + tags=["Customer Management"], + include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) async def update_end_user(): @@ -7897,9 +7930,15 @@ async def update_end_user(): pass +@router.post( + "/customer/delete", + tags=["Customer Management"], + dependencies=[Depends(user_api_key_auth)], +) @router.post( "/end_user/delete", - tags=["End User Management"], + tags=["Customer Management"], + include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) async def delete_end_user(): From 8792b8c7fa92b51cdd02d89621cba237c8296511 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 18:18:58 -0700 Subject: [PATCH 05/12] docs - rename end user -> customer --- docs/my-website/docs/proxy/users.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/my-website/docs/proxy/users.md b/docs/my-website/docs/proxy/users.md index 556ae7f92fb..ec2be9cdc77 100644 --- a/docs/my-website/docs/proxy/users.md +++ b/docs/my-website/docs/proxy/users.md @@ -13,7 +13,7 @@ Requirements: You can set budgets at 3 levels: - For the proxy - For an internal user -- For an end-user +- For a customer (end-user) - For a key - For a key (model specific budgets) @@ -173,7 +173,7 @@ curl --location 'http://localhost:4000/chat/completions' \ ``` - + Use this to budget `user` passed to `/chat/completions`, **without needing to create a key for every user** @@ -452,7 +452,7 @@ curl --location 'http://0.0.0.0:4000/key/generate' \ ``` - + :::info @@ -477,12 +477,12 @@ curl --location 'http://0.0.0.0:4000/budget/new' \ ``` -#### Step 2. Create `End-User` with Budget +#### Step 2. Create `Customer` with Budget -We use `budget_id="free-tier"` from Step 1 when creating this new end user +We use `budget_id="free-tier"` from Step 1 when creating this new customers ```shell -curl --location 'http://0.0.0.0:4000/end_user/new' \ +curl --location 'http://0.0.0.0:4000/customer/new' \ --header 'Authorization: Bearer sk-1234' \ --header 'Content-Type: application/json' \ --data '{ @@ -492,7 +492,7 @@ curl --location 'http://0.0.0.0:4000/end_user/new' \ ``` -#### Step 3. Pass end user id in `/chat/completions` requests +#### Step 3. Pass `user_id` id in `/chat/completions` requests Pass the `user_id` from Step 2 as `user="palantir"` From cdf32ebf0ec3ae66eacc317538a1d5aa5a70a1a1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 18:20:44 -0700 Subject: [PATCH 06/12] docs string - > end user /new --- litellm/proxy/proxy_server.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b7985ce5ba9..ea1677aa393 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7796,9 +7796,8 @@ async def new_end_user( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ - [TODO] Needs to be implemented. - - Allow creating a new end-user + Allow creating a new Customer + NOTE: This used to be called `/end_user/new`, we will still be maintaining compatibility for /end_user/XXX for these endpoints - Allow specifying allowed regions - Allow specifying default model From 24f0b82755b1262c663157d60642827a669cf5ef Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 18:29:09 -0700 Subject: [PATCH 07/12] feat - add validation for existing customers --- litellm/proxy/proxy_server.py | 102 +++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 38 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ea1677aa393..b22bbd291c9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7833,51 +7833,77 @@ async def new_end_user( status_code=500, detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) + try: - ## VALIDATION ## - if data.default_model is not None: - if llm_router is None: - raise HTTPException( - status_code=422, detail={"error": CommonProxyErrors.no_llm_router.value} - ) - elif data.default_model not in llm_router.get_model_names(): - raise HTTPException( - status_code=422, - detail={ - "error": "Default Model not on proxy. Configure via `/model/new` or config.yaml. Default_model={}, proxy_model_names={}".format( - data.default_model, set(llm_router.get_model_names()) - ) - }, + ## VALIDATION ## + if data.default_model is not None: + if llm_router is None: + raise HTTPException( + status_code=422, + detail={"error": CommonProxyErrors.no_llm_router.value}, + ) + elif data.default_model not in llm_router.get_model_names(): + raise HTTPException( + status_code=422, + detail={ + "error": "Default Model not on proxy. Configure via `/model/new` or config.yaml. Default_model={}, proxy_model_names={}".format( + data.default_model, set(llm_router.get_model_names()) + ) + }, + ) + + new_end_user_obj: Dict = {} + + ## CREATE BUDGET ## if set + if data.max_budget is not None: + budget_record = await prisma_client.db.litellm_budgettable.create( + data={ + "max_budget": data.max_budget, + "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, # type: ignore + "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, + } ) - new_end_user_obj: Dict = {} + new_end_user_obj["budget_id"] = budget_record.budget_id + elif data.budget_id is not None: + new_end_user_obj["budget_id"] = data.budget_id - ## CREATE BUDGET ## if set - if data.max_budget is not None: - budget_record = await prisma_client.db.litellm_budgettable.create( - data={ - "max_budget": data.max_budget, - "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name, # type: ignore - "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name, - } + _user_data = data.dict(exclude_none=True) + + for k, v in _user_data.items(): + if k != "max_budget" and k != "budget_id": + new_end_user_obj[k] = v + + ## WRITE TO DB ## + end_user_record = await prisma_client.db.litellm_endusertable.create( + data=new_end_user_obj # type: ignore ) - new_end_user_obj["budget_id"] = budget_record.budget_id - elif data.budget_id is not None: - new_end_user_obj["budget_id"] = data.budget_id + return end_user_record + except Exception as e: + if "Unique constraint failed on the fields: (`user_id`)" in str(e): + raise ProxyException( + message=f"Customer already exists, passed user_id={data.user_id}. Please pass a new user_id.", + type="bad_request", + code=400, + param="user_id", + ) - _user_data = data.dict(exclude_none=True) - - for k, v in _user_data.items(): - if k != "max_budget" and k != "budget_id": - new_end_user_obj[k] = v - - ## WRITE TO DB ## - end_user_record = await prisma_client.db.litellm_endusertable.create( - data=new_end_user_obj # type: ignore - ) - - return end_user_record + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Internal Server Error({str(e)})"), + type="internal_error", + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Internal Server Error, " + str(e), + type="internal_error", + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) @router.get( From f5886164988f3baa3e33f285bd8efb7c7147b050 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 19:02:20 -0700 Subject: [PATCH 08/12] fix - /customer/update --- litellm/proxy/_types.py | 14 +++++++ litellm/proxy/proxy_server.py | 77 +++++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e85f116f73b..d93c1e33b27 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -540,6 +540,20 @@ class NewEndUserRequest(LiteLLMBase): return values +class UpdateEndUserRequest(LiteLLMBase): + user_id: str + alias: Optional[str] = None # human-friendly alias + blocked: bool = False # allow/disallow requests for this end-user + max_budget: Optional[float] = None + budget_id: Optional[str] = None # give either a budget_id or max_budget + allowed_model_region: Optional[Literal["eu"]] = ( + None # require all user requests to use models in this specific region + ) + default_model: Optional[str] = ( + None # if no equivalent model in allowed region - default all requests to this model + ) + + class Member(LiteLLMBase): role: Literal["admin", "user"] user_id: Optional[str] = None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b22bbd291c9..66a1d5e1607 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7809,7 +7809,8 @@ async def new_end_user( --header 'Content-Type: application/json' \ --data '{ "user_id" : "ishaan-jaff-3", - "allowed_region": "eu" + "allowed_region": "eu", + "budget_id": "free_tier", "default_model": "azure/gpt-3.5-turbo-eu" <- all calls from this user, use this model? }' @@ -7948,10 +7949,80 @@ async def end_user_info( include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) -async def update_end_user(): +async def update_end_user( + data: UpdateEndUserRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ - [TODO] Needs to be implemented. + Example curl + + ``` + curl --location 'http://0.0.0.0:4000/customer/update' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "user_id": "test-litellm-user-4", + "budget_id": "paid_tier" + }' + + See below for all params + ``` """ + + global prisma_client + try: + data_json: dict = data.json() + # get the row from db + if prisma_client is None: + raise Exception("Not connected to DB!") + + # get non default values for key + non_default_values = {} + for k, v in data_json.items(): + if v is not None and v not in ( + [], + {}, + 0, + ): # models default to [], spend defaults to 0, we should not reset these values + non_default_values[k] = v + + ## ADD USER, IF NEW ## + verbose_proxy_logger.debug("/customer/update: Received data = %s", data) + if data.user_id is not None and len(data.user_id) > 0: + non_default_values["user_id"] = data.user_id # type: ignore + verbose_proxy_logger.debug("In update customer, user_id condition block.") + response = await prisma_client.db.litellm_endusertable.update( + where={"user_id": data.user_id}, data=non_default_values # type: ignore + ) + if response is None: + raise ValueError( + f"Failed updating customer data. User ID does not exist passed user_id={data.user_id}" + ) + verbose_proxy_logger.debug( + f"received response from updating prisma client. response={response}" + ) + return response + else: + raise ValueError(f"user_id is required, passed user_id = {data.user_id}") + + # update based on remaining passed in values + except Exception as e: + traceback.print_exc() + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Internal Server Error({str(e)})"), + type="internal_error", + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Internal Server Error, " + str(e), + type="internal_error", + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) pass From 0feeb53868051cd0e3424dbbcdc476663dc9c4d8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 19:24:20 -0700 Subject: [PATCH 09/12] fix - working customer/delete --- litellm/proxy/_types.py | 21 +++++++++- litellm/proxy/proxy_server.py | 74 +++++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d93c1e33b27..13d83cbc2d6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -519,7 +519,11 @@ class UpdateUserRequest(GenerateRequestBase): return values -class NewEndUserRequest(LiteLLMBase): +class NewCustomerRequest(LiteLLMBase): + """ + Create a new customer, allocate a budget to them + """ + user_id: str alias: Optional[str] = None # human-friendly alias blocked: bool = False # allow/disallow requests for this end-user @@ -540,7 +544,12 @@ class NewEndUserRequest(LiteLLMBase): return values -class UpdateEndUserRequest(LiteLLMBase): +class UpdateCustomerRequest(LiteLLMBase): + """ + Update a Customer, use this to update customer budgets etc + + """ + user_id: str alias: Optional[str] = None # human-friendly alias blocked: bool = False # allow/disallow requests for this end-user @@ -554,6 +563,14 @@ class UpdateEndUserRequest(LiteLLMBase): ) +class DeleteCustomerRequest(LiteLLMBase): + """ + Delete multiple Customers + """ + + user_ids: List[str] + + class Member(LiteLLMBase): role: Literal["admin", "user"] user_id: Optional[str] = None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 66a1d5e1607..0778e678a7b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7792,7 +7792,7 @@ async def unblock_user(data: BlockUsers): dependencies=[Depends(user_api_key_auth)], ) async def new_end_user( - data: NewEndUserRequest, + data: NewCustomerRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -7950,7 +7950,7 @@ async def end_user_info( dependencies=[Depends(user_api_key_auth)], ) async def update_end_user( - data: UpdateEndUserRequest, + data: UpdateCustomerRequest, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -8037,10 +8037,76 @@ async def update_end_user( include_in_schema=False, dependencies=[Depends(user_api_key_auth)], ) -async def delete_end_user(): +async def delete_end_user( + data: DeleteCustomerRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): """ - [TODO] Needs to be implemented. + Example curl + + ``` + curl --location 'http://0.0.0.0:4000/customer/delete' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "user_ids" :["ishaan-jaff-5"] + }' + + See below for all params + ``` """ + global prisma_client + + try: + if prisma_client is None: + raise Exception("Not connected to DB!") + + verbose_proxy_logger.debug("/customer/delete: Received data = %s", data) + if ( + data.user_ids is not None + and isinstance(data.user_ids, list) + and len(data.user_ids) > 0 + ): + response = await prisma_client.db.litellm_endusertable.delete_many( + where={"user_id": {"in": data.user_ids}} + ) + if response is None: + raise ValueError( + f"Failed deleting customer data. User ID does not exist passed user_id={data.user_ids}" + ) + if response != len(data.user_ids): + raise ValueError( + f"Failed deleting all customer data. User ID does not exist passed user_id={data.user_ids}. Deleted {response} customers, passed {len(data.user_ids)} customers" + ) + verbose_proxy_logger.debug( + f"received response from updating prisma client. response={response}" + ) + return { + "deleted_customers": response, + "message": "Successfully deleted customers with ids: " + + str(data.user_ids), + } + else: + raise ValueError(f"user_id is required, passed user_id = {data.user_ids}") + + # update based on remaining passed in values + except Exception as e: + traceback.print_exc() + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Internal Server Error({str(e)})"), + type="internal_error", + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Internal Server Error, " + str(e), + type="internal_error", + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) pass From 7d5fe910f20fd03a0711ad496951602bf1499b2a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 19:27:20 -0700 Subject: [PATCH 10/12] fix - make email alerting free --- litellm/proxy/utils.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b01312478b6..b96c469ec09 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1850,20 +1850,13 @@ async def send_email(receiver_email, subject, html): from litellm.proxy.proxy_server import premium_user from litellm.proxy.proxy_server import CommonProxyErrors - # Check if user is premium - This is an Enterprise only Feature - if premium_user != True: - raise Exception( - f"Trying to use Email Alerting\n {CommonProxyErrors.not_premium_user.value}" - ) - # Done Checking - smtp_host = os.getenv("SMTP_HOST") - smtp_port = os.getenv("SMTP_PORT", 587) # default to port 587 + smtp_port = int(os.getenv("SMTP_PORT", "587")) # default to port 587 smtp_username = os.getenv("SMTP_USERNAME") smtp_password = os.getenv("SMTP_PASSWORD") sender_email = os.getenv("SMTP_SENDER_EMAIL", None) if sender_email is None: - raise Exception("Trying to use SMTP, but SMTP_SENDER_EMAIL is not set") + raise ValueError("Trying to use SMTP, but SMTP_SENDER_EMAIL is not set") ## EMAIL SETUP ## email_message = MIMEMultipart() From 528003bc6103f5427d6ce67bd044760310d36c2d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 19:38:42 -0700 Subject: [PATCH 11/12] docs - Email Notifications --- docs/my-website/docs/proxy/email.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/my-website/docs/proxy/email.md b/docs/my-website/docs/proxy/email.md index 62dfeccf7ab..2551f4359b1 100644 --- a/docs/my-website/docs/proxy/email.md +++ b/docs/my-website/docs/proxy/email.md @@ -2,12 +2,6 @@ import Image from '@theme/IdealImage'; # ✨ 📧 Email Notifications -:::info - -This is an Enterprise only feature [Get in touch with us for a Free Trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) - -::: - Send an Email to your users when: - A Proxy API Key is created for them - Their API Key crosses it's Budget @@ -38,6 +32,12 @@ That's it ! start your proxy ## Customizing Email Branding +:::info + +Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + +::: + LiteLLM allows you to customize the: - Logo on the Email - Email support contact From 8d0bd9d8f10c8c0c044c65b1130aec6a4498f0e7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 May 2024 19:40:47 -0700 Subject: [PATCH 12/12] ffeat - make email alerting a free feature --- litellm/integrations/slack_alerting.py | 49 +++++++++++++++++--------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 9e35b4fc3d7..c822958d93f 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -41,10 +41,6 @@ class ProviderRegionOutageModel(BaseOutageModel): # we use this for the email header, please send a test email if you change this. verify it looks good on email LITELLM_LOGO_URL = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" -EMAIL_LOGO_URL = os.getenv( - "SMTP_SENDER_LOGO", "https://litellm-listing.s3.amazonaws.com/litellm_logo.png" -) -EMAIL_SUPPORT_CONTACT = os.getenv("EMAIL_SUPPORT_CONTACT", "support@berri.ai") class LiteLLMBase(BaseModel): @@ -1147,21 +1143,34 @@ Model Info: return False + async def _check_if_using_premium_email_feature( + self, + premium_user: bool, + email_logo_url: Optional[str] = None, + email_support_contact: Optional[str] = None, + ): + from litellm.proxy.proxy_server import premium_user + from litellm.proxy.proxy_server import CommonProxyErrors + + if premium_user is not True: + if email_logo_url is not None or email_support_contact is not None: + raise ValueError( + f"Trying to Customize Email Alerting\n {CommonProxyErrors.not_premium_user.value}" + ) + async def send_key_created_email(self, webhook_event: WebhookEvent) -> bool: from litellm.proxy.utils import send_email if self.alerting is None or "email" not in self.alerting: # do nothing if user does not want email alerts return False + from litellm.proxy.proxy_server import premium_user, prisma_client - # make sure this is a premium user - from litellm.proxy.proxy_server import premium_user - from litellm.proxy.proxy_server import CommonProxyErrors, prisma_client - - if premium_user != True: - raise Exception( - f"Trying to use Email Alerting on key creation\n {CommonProxyErrors.not_premium_user.value}" - ) + email_logo_url = os.getenv("SMTP_SENDER_LOGO", None) + email_support_contact = os.getenv("EMAIL_SUPPORT_CONTACT", None) + await self._check_if_using_premium_email_feature( + premium_user, email_logo_url, email_support_contact + ) event_name = webhook_event.event_message recipient_email = webhook_event.user_email @@ -1188,7 +1197,7 @@ Model Info: "Trying to send email alert to no recipient", extra=webhook_event.dict() ) email_html_content = f""" - LiteLLM Logo + LiteLLM Logo

Hi {recipient_email},
@@ -1223,7 +1232,7 @@ Model Info: - If you have any questions, please send an email to {EMAIL_SUPPORT_CONTACT}

+ If you have any questions, please send an email to {email_support_contact}

Best,
The LiteLLM team
@@ -1254,6 +1263,14 @@ Model Info: """ from litellm.proxy.utils import send_email + from litellm.proxy.proxy_server import premium_user, prisma_client + + email_logo_url = os.getenv("SMTP_SENDER_LOGO", None) + email_support_contact = os.getenv("EMAIL_SUPPORT_CONTACT", None) + await self._check_if_using_premium_email_feature( + premium_user, email_logo_url, email_support_contact + ) + event_name = webhook_event.event_message recipient_email = webhook_event.user_email user_name = webhook_event.user_id @@ -1266,7 +1283,7 @@ Model Info: if webhook_event.event == "budget_crossed": email_html_content = f""" - LiteLLM Logo + LiteLLM Logo

Hi {user_name},
@@ -1274,7 +1291,7 @@ Model Info: API requests will be rejected until either (a) you increase your monthly budget or (b) your monthly usage resets at the beginning of the next calendar month.

- If you have any questions, please send an email to {EMAIL_SUPPORT_CONTACT}

+ If you have any questions, please send an email to {email_support_contact}

Best,
The LiteLLM team