From 1f4f1c6f709486f6059fe11f8d618861d3d9cc45 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 14:19:23 -0700 Subject: [PATCH 01/14] stash /model/metrics/exceptions endpoints --- litellm/proxy/proxy_server.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3a7821d2727..b3132d2c641 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7596,6 +7596,41 @@ async def model_metrics( return response +@router.get( + "/model/metrics/exceptions", + description="View number of failed requests per model on config.yaml", + tags=["model management"], + include_in_schema=False, + dependencies=[Depends(user_api_key_auth)], +) +async def model_metrics_exceptions( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + _selected_model_group: Optional[str] = None, + startTime: Optional[datetime] = datetime.now() - timedelta(days=30), + endTime: Optional[datetime] = datetime.now(), +): + global prisma_client, llm_router + if prisma_client is None: + raise ProxyException( + message="Prisma Client is not initialized", + type="internal_error", + param="None", + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + sql_query = """ + SELECT model_group, api_base, exception_type, COUNT(*) AS num_exceptions + FROM "LiteLLM_ErrorLogs" + WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp + GROUP BY model_group, api_base, exception_type + ORDER BY num_exceptions DESC + LIMIT 50; + """ + db_response = await prisma_client.db.query_raw(sql_query, startTime, endTime) + response: List[dict] = [] + return response + + @router.get( "/model/info", description="Provides more info about each model in /models, including config.yaml descriptions (except api key and api base)", From b9a0a13516b9682f6c40be97bea0f25aaad4fab6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 14:28:19 -0700 Subject: [PATCH 02/14] ui - show model usage --- .../src/components/model_dashboard.tsx | 102 ++++++++++++++++-- 1 file changed, 95 insertions(+), 7 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 2491b40e627..a48736f14b2 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -18,7 +18,7 @@ import { } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react"; import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react"; -import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall } from "./networking"; +import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall } from "./networking"; import { BarChart } from "@tremor/react"; import { Button as Button2, @@ -200,6 +200,8 @@ const ModelDashboard: React.FC = ({ const [selectedModel, setSelectedModel] = useState(null); const [availableModelGroups, setAvailableModelGroups] = useState>([]); const [selectedModelGroup, setSelectedModelGroup] = useState(null); + const [modelLatencyMetrics, setModelLatencyMetrics] = useState([]); + const [modelMetrics, setModelMetrics] = useState([]); const EditModelModal: React.FC = ({ visible, onCancel, model, onSubmit }) => { const [form] = Form.useForm(); @@ -445,12 +447,20 @@ const handleEditSubmit = async (formValues: Record) => { let _array_model_groups = Array.from(all_model_groups) setAvailableModelGroups(_array_model_groups); - // if userRole is Admin, show the pending requests - if (userRole === "Admin" && accessToken) { - const user_requests = await userGetRequesedtModelsCall(accessToken); - console.log("Pending Requests:", pendingRequests); - setPendingRequests(user_requests.requests || []); - } + const modelMetricsResponse = await modelMetricsCall( + accessToken, + userID, + userRole, + null + ); + + console.log("Model metrics response:", modelMetricsResponse); + // Sort by latency (avg_latency_seconds) + const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_seconds - a.avg_latency_seconds); + console.log("Sorted by latency:", sortedByLatency); + + setModelMetrics(modelMetricsResponse); + setModelLatencyMetrics(sortedByLatency); } catch (error) { console.error("There was an error fetching the model data", error); } @@ -603,6 +613,30 @@ const handleEditSubmit = async (formValues: Record) => { }; + const updateModelMetrics = async (modelGroup: string | null) => { + console.log("Updating model metrics for group:", modelGroup); + if (!accessToken || !userID || !userRole) { + return + } + setSelectedModelGroup(modelGroup); // If you want to store the selected model group in state + + + try { + const modelMetricsResponse = await modelMetricsCall(accessToken, userID, userRole, modelGroup); + console.log("Model metrics response:", modelMetricsResponse); + + // Assuming modelMetricsResponse now contains the metric data for the specified model group + const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_seconds - a.avg_latency_seconds); + console.log("Sorted by latency:", sortedByLatency); + + setModelMetrics(modelMetricsResponse); + setModelLatencyMetrics(sortedByLatency); + } catch (error) { + console.error("Failed to fetch model metrics", error); + } + } + + const getPlaceholder = (selectedProvider: string): string => { if (selectedProvider === Providers.Vertex_AI) { @@ -639,6 +673,7 @@ const handleEditSubmit = async (formValues: Record) => {
All Models Add Model + Model Analytics 
/health Models
@@ -944,6 +979,59 @@ const handleEditSubmit = async (formValues: Record) => { + +

View how requests were load balanced within a model group

+

(Beta feature) only supported for Azure Model Groups

+ + + + + Number Requests per Model + + + + Latency Per Model + + + + `/health` will run a very small request through your models configured on litellm From 49f83ce2045fa2f0312995c59b238fd143b5a608 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 15:16:25 -0700 Subject: [PATCH 03/14] ui - show models analytics --- ui/litellm-dashboard/src/components/usage.tsx | 118 ------------------ 1 file changed, 118 deletions(-) diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/components/usage.tsx index 0caeaccc468..f0b4f0551a7 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/components/usage.tsx @@ -146,10 +146,6 @@ const UsagePage: React.FC = ({ const [topTagsData, setTopTagsData] = useState([]); const [uniqueTeamIds, setUniqueTeamIds] = useState([]); const [totalSpendPerTeam, setTotalSpendPerTeam] = useState([]); - const [modelMetrics, setModelMetrics] = useState([]); - const [modelLatencyMetrics, setModelLatencyMetrics] = useState([]); - const [modelGroups, setModelGroups] = useState([]); - const [selectedModelGroup, setSelectedModelGroup] = useState(null); const firstDay = new Date( currentDate.getFullYear(), @@ -231,25 +227,6 @@ const UsagePage: React.FC = ({ const top_tags = await tagsSpendLogsCall(accessToken); setTopTagsData(top_tags.top_10_tags); - // get model groups - const _model_groups = await modelInfoCall(accessToken, userID, userRole); - let model_groups = _model_groups.data; - console.log("model groups in model dashboard", model_groups); - - let available_model_groups = []; - // loop through each model in model_group, access litellm_params and only inlclude the model if model["litellm_params"]["model"] startswith "azure/" - for (let i = 0; i < model_groups.length; i++) { - let model = model_groups[i]; - console.log("model check", model); - let model_group = model["litellm_params"]["model"]; - console.log("model group", model_group); - if (model_group.startsWith("azure/")) { - available_model_groups.push(model["model_name"]); - } - } - setModelGroups(available_model_groups); - - } else if (userRole == "App Owner") { await userSpendLogsCall( accessToken, @@ -286,22 +263,6 @@ const UsagePage: React.FC = ({ } }); } - - const modelMetricsResponse = await modelMetricsCall( - accessToken, - userID, - userRole, - null - ); - - console.log("Model metrics response:", modelMetricsResponse); - // Sort by latency (avg_latency_seconds) - const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_seconds - a.avg_latency_seconds); - console.log("Sorted by latency:", sortedByLatency); - - setModelMetrics(modelMetricsResponse); - setModelLatencyMetrics(sortedByLatency); - } catch (error) { console.error("There was an error fetching the data", error); // Optionally, update your UI to reflect the error state here as well @@ -312,30 +273,6 @@ const UsagePage: React.FC = ({ }, [accessToken, token, userRole, userID, startTime, endTime]); - const updateModelMetrics = async (modelGroup: string | null) => { - console.log("Updating model metrics for group:", modelGroup); - if (!accessToken || !userID || !userRole) { - return - } - setSelectedModelGroup(modelGroup); // If you want to store the selected model group in state - - - try { - const modelMetricsResponse = await modelMetricsCall(accessToken, userID, userRole, modelGroup); - console.log("Model metrics response:", modelMetricsResponse); - - // Assuming modelMetricsResponse now contains the metric data for the specified model group - const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_seconds - a.avg_latency_seconds); - console.log("Sorted by latency:", sortedByLatency); - - setModelMetrics(modelMetricsResponse); - setModelLatencyMetrics(sortedByLatency); - } catch (error) { - console.error("Failed to fetch model metrics", error); - } - } - - return (
= ({ All Up Team Based Usage Tag Based Usage - Model Based Usage @@ -492,60 +428,6 @@ const UsagePage: React.FC = ({ - - Filter By Model Group -

View how requests were load balanced within a model group

-

(Beta feature) only supported for Azure Model Groups

- - - - - Number Requests per Model - - - - Latency Per Model - - - -
From 26a5d85869e88db9b7988ec8c5d79a9d23b33c90 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 15:41:16 -0700 Subject: [PATCH 04/14] fix - backend return exceptions --- litellm/proxy/proxy_server.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b3132d2c641..37c943636c2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7624,10 +7624,28 @@ async def model_metrics_exceptions( WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp GROUP BY model_group, api_base, exception_type ORDER BY num_exceptions DESC - LIMIT 50; + LIMIT 200; """ db_response = await prisma_client.db.query_raw(sql_query, startTime, endTime) response: List[dict] = [] + for model_data in db_response: + model = model_data.get("model_group", "") + api_base = model_data.get("api_base", "") + exception_type = model_data.get("exception_type", "") + num_exceptions = model_data.get("num_exceptions", 0) + + response.append( + { + "model_group": model + "-" + api_base, + "exception_type": exception_type, + "num_exceptions": num_exceptions, + } + ) + + # sort all entries in descending order based on num_exceptions + + response.sort(key=lambda x: x["num_exceptions"], reverse=True) + return response From a2a8fef8f4f811284c10635cae864d81171ada71 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 16:53:53 -0700 Subject: [PATCH 05/14] fix passing starttime and endtime to model/exceptions --- litellm/proxy/proxy_server.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 37c943636c2..608049793be 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7606,8 +7606,8 @@ async def model_metrics( async def model_metrics_exceptions( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), _selected_model_group: Optional[str] = None, - startTime: Optional[datetime] = datetime.now() - timedelta(days=30), - endTime: Optional[datetime] = datetime.now(), + startTime: Optional[datetime] = None, + endTime: Optional[datetime] = None, ): global prisma_client, llm_router if prisma_client is None: @@ -7618,6 +7618,9 @@ async def model_metrics_exceptions( code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + startTime = startTime or datetime.now() - timedelta(days=30) + endTime = endTime or datetime.now() + sql_query = """ SELECT model_group, api_base, exception_type, COUNT(*) AS num_exceptions FROM "LiteLLM_ErrorLogs" @@ -7628,25 +7631,29 @@ async def model_metrics_exceptions( """ db_response = await prisma_client.db.query_raw(sql_query, startTime, endTime) response: List[dict] = [] + exception_types = set() for model_data in db_response: model = model_data.get("model_group", "") api_base = model_data.get("api_base", "") exception_type = model_data.get("exception_type", "") num_exceptions = model_data.get("num_exceptions", 0) + exception_types.add(exception_type) response.append( { "model_group": model + "-" + api_base, - "exception_type": exception_type, + exception_type: num_exceptions, "num_exceptions": num_exceptions, } ) # sort all entries in descending order based on num_exceptions + response = sorted(response, key=lambda x: x["num_exceptions"], reverse=True) - response.sort(key=lambda x: x["num_exceptions"], reverse=True) - - return response + return { + "data": response, + "exception_types": list(exception_types), + } @router.get( From ce1817380e3412081f0991afac752ce5b7432b69 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 16:56:45 -0700 Subject: [PATCH 06/14] feat ui - modelExceptionsCall --- .../src/components/model_dashboard.tsx | 29 ++++++++++++++- .../src/components/networking.tsx | 37 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index a48736f14b2..c63f6ed9367 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -18,7 +18,7 @@ import { } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react"; import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react"; -import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall } from "./networking"; +import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall } from "./networking"; import { BarChart } from "@tremor/react"; import { Button as Button2, @@ -202,6 +202,8 @@ const ModelDashboard: React.FC = ({ const [selectedModelGroup, setSelectedModelGroup] = useState(null); const [modelLatencyMetrics, setModelLatencyMetrics] = useState([]); const [modelMetrics, setModelMetrics] = useState([]); + const [modelExceptions, setModelExceptions] = useState([]); + const [allExceptions, setAllExceptions] = useState([]); const EditModelModal: React.FC = ({ visible, onCancel, model, onSubmit }) => { const [form] = Form.useForm(); @@ -461,6 +463,17 @@ const handleEditSubmit = async (formValues: Record) => { setModelMetrics(modelMetricsResponse); setModelLatencyMetrics(sortedByLatency); + + const modelExceptionsResponse = await modelExceptionsCall( + accessToken, + userID, + userRole, + null + ) + console.log("Model exceptions response:", modelExceptionsResponse); + setModelExceptions(modelExceptionsResponse.data); + setAllExceptions(modelExceptionsResponse.exception_types); + } catch (error) { console.error("There was an error fetching the model data", error); } @@ -1017,6 +1030,20 @@ const handleEditSubmit = async (formValues: Record) => { tickGap={5} /> + + + Exceptions per Model + + + Latency Per Model { + /** + * Get all models on proxy + */ + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics/exceptions` : `/model/metrics/exceptions`; + + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData, 20); + throw new Error("Network response was not ok"); + } + const data = await response.json(); + // message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + + export const modelAvailableCall = async ( accessToken: String, userID: String, From 8177ef5ec00c89c6dbca834d026ad2ea63959996 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 17:23:27 -0700 Subject: [PATCH 07/14] ui - show model latency / token --- .../src/components/model_dashboard.tsx | 84 ++++++++++--------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index c63f6ed9367..9d1c6b6c2e1 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -457,8 +457,8 @@ const handleEditSubmit = async (formValues: Record) => { ); console.log("Model metrics response:", modelMetricsResponse); - // Sort by latency (avg_latency_seconds) - const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_seconds - a.avg_latency_seconds); + // Sort by latency (avg_latency_per_token) + const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_per_token - a.avg_latency_per_token); console.log("Sorted by latency:", sortedByLatency); setModelMetrics(modelMetricsResponse); @@ -686,8 +686,8 @@ const handleEditSubmit = async (formValues: Record) => {
All Models Add Model - Model Analytics 
/health Models
 + Model Analytics
@@ -992,6 +992,17 @@ const handleEditSubmit = async (formValues: Record) => { + + + `/health` will run a very small request through your models configured on litellm + + + {healthCheckResponse && ( + 
{JSON.stringify(healthCheckResponse, null, 2)}
+ )} + + +

View how requests were load balanced within a model group

(Beta feature) only supported for Azure Model Groups

@@ -1017,20 +1028,33 @@ const handleEditSubmit = async (formValues: Record) => { ))} - - Number Requests per Model - - - + + + + + + + Model + Median Latency/Token + + + + {modelLatencyMetrics.map((metric, idx) => ( + + {metric.model} + {metric.avg_latency_per_token.toFixed(4)} + + ))} + +
+ + + + + Requests, Failures per Model + + + Exceptions per Model ) => { /> - - Latency Per Model - - + - - - `/health` will run a very small request through your models configured on litellm - - - {healthCheckResponse && ( - 
{JSON.stringify(healthCheckResponse, null, 2)}
- )} - - - + From 8a1a043801fe806d51190cb0ee02b77940e68fa5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 17:23:36 -0700 Subject: [PATCH 08/14] backend - show model latency per token --- litellm/proxy/proxy_server.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 608049793be..a970b690391 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7524,8 +7524,8 @@ async def model_info_v2( async def model_metrics( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), _selected_model_group: Optional[str] = None, - startTime: Optional[datetime] = datetime.now() - timedelta(days=30), - endTime: Optional[datetime] = datetime.now(), + startTime: Optional[datetime] = None, + endTime: Optional[datetime] = None, ): global prisma_client, llm_router if prisma_client is None: @@ -7535,6 +7535,8 @@ async def model_metrics( param="None", code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + startTime = startTime or datetime.now() - timedelta(days=30) + endTime = endTime or datetime.now() if _selected_model_group and llm_router is not None: _model_list = llm_router.get_model_list() _relevant_api_bases = [] @@ -7567,15 +7569,11 @@ async def model_metrics( SELECT CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base, COUNT(*) AS num_requests, - AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) AS avg_latency_seconds - FROM - "LiteLLM_SpendLogs" + AVG(EXTRACT(epoch FROM ("endTime" - "startTime")) / total_tokens) AS avg_latency_per_token + FROM "LiteLLM_SpendLogs" WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp - GROUP BY - CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END - ORDER BY - num_requests DESC - LIMIT 50; + GROUP BY CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END + ORDER BY num_requests DESC; """ db_response = await prisma_client.db.query_raw(sql_query, startTime, endTime) @@ -7585,12 +7583,12 @@ async def model_metrics( for model_data in db_response: model = model_data.get("combined_model_api_base", "") num_requests = model_data.get("num_requests", 0) - avg_latency_seconds = model_data.get("avg_latency_seconds", 0) + avg_latency_per_token = model_data.get("avg_latency_per_token", 0) response.append( { "model": model, "num_requests": num_requests, - "avg_latency_seconds": avg_latency_seconds, + "avg_latency_per_token": avg_latency_per_token, } ) return response From f2849d06415820d095ed16391c4c6880bc2c3364 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 17:31:40 -0700 Subject: [PATCH 09/14] fix - track litellm_model_name in LiteLLM_ErrorLogs --- litellm/proxy/_types.py | 1 + litellm/proxy/proxy_server.py | 1 + schema.prisma | 1 + 3 files changed, 3 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c910664f15e..b7b2d0ab6b5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -916,6 +916,7 @@ class LiteLLM_ErrorLogs(LiteLLMBase): request_id: Optional[str] = str(uuid.uuid4()) api_base: Optional[str] = "" model_group: Optional[str] = "" + litellm_model_name: Optional[str] = "" model_id: Optional[str] = "" request_kwargs: Optional[dict] = {} exception_type: Optional[str] = "" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a970b690391..166422fbd9e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1258,6 +1258,7 @@ async def _PROXY_failure_handler( request_id=str(uuid.uuid4()), model_group=_model_group, model_id=_model_id, + litellm_model_name=kwargs.get("model"), request_kwargs=_optional_params, api_base=api_base, exception_type=_exception_type, diff --git a/schema.prisma b/schema.prisma index b362a0ec025..f078dbcf432 100644 --- a/schema.prisma +++ b/schema.prisma @@ -190,6 +190,7 @@ model LiteLLM_ErrorLogs { endTime DateTime // Assuming end_time is a DateTime field api_base String @default("") model_group String @default("") // public model_name / model_group + litellm_model_name String @default("") // model passed to litellm model_id String @default("") // ID of model in ProxyModelTable request_kwargs Json @default("{}") exception_type String @default("") From ace3b02d9705f9a87ec9bffa67e1b1aecd98cef4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 18:23:31 -0700 Subject: [PATCH 10/14] ui - model analytics show failed requests % --- .../src/components/model_dashboard.tsx | 61 ++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 9d1c6b6c2e1..cb2246bad24 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -204,6 +204,7 @@ const ModelDashboard: React.FC = ({ const [modelMetrics, setModelMetrics] = useState([]); const [modelExceptions, setModelExceptions] = useState([]); const [allExceptions, setAllExceptions] = useState([]); + const [failureTableData, setFailureTableData] = useState([]); const EditModelModal: React.FC = ({ visible, onCancel, model, onSubmit }) => { const [form] = Form.useForm(); @@ -474,6 +475,39 @@ const handleEditSubmit = async (formValues: Record) => { setModelExceptions(modelExceptionsResponse.data); setAllExceptions(modelExceptionsResponse.exception_types); + let successdeploymentToSuccess: Record = {}; + for (let i = 0; i < modelMetricsResponse.length; i++) { + let element = modelMetricsResponse[i]; + let _model_name = element.model; + let _num_requests = element.num_requests; + successdeploymentToSuccess[_model_name] = _num_requests + } + console.log("successdeploymentToSuccess:", successdeploymentToSuccess) + + let failureTableData = []; + let _failureData = modelExceptionsResponse.data; + for (let i = 0; i < _failureData.length; i++) { + const model = _failureData[i]; + let _model_name = model.model; + let total_exceptions = model.total_exceptions; + let total_Requests = successdeploymentToSuccess[_model_name]; + if (total_Requests == null) { + total_Requests = 0 + } + let _data = { + model: _model_name, + total_exceptions: total_exceptions, + total_Requests: total_Requests, + failure_rate: total_Requests / total_exceptions + } + failureTableData.push(_data); + // sort failureTableData by failure_rate + failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); + + setFailureTableData(failureTableData); + console.log("failureTableData:", failureTableData); + } + } catch (error) { console.error("There was an error fetching the model data", error); } @@ -1051,7 +1085,30 @@ const handleEditSubmit = async (formValues: Record) => { - Requests, Failures per Model + + + + Model + Success Requests + Error Requests + Failure % + + + + + {failureTableData.map((metric, idx) => ( + + {metric.model} + {metric.total_Requests} + {metric.total_exceptions} + {metric.failure_rate}% + + + ))} + +
+ + @@ -1060,7 +1117,7 @@ const handleEditSubmit = async (formValues: Record) => { Date: Tue, 30 Apr 2024 18:26:14 -0700 Subject: [PATCH 11/14] fix - viewing model metrics --- litellm/proxy/proxy_server.py | 68 ++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 166422fbd9e..3bc9e7d52c9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7620,39 +7620,57 @@ async def model_metrics_exceptions( startTime = startTime or datetime.now() - timedelta(days=30) endTime = endTime or datetime.now() + """ + """ sql_query = """ - SELECT model_group, api_base, exception_type, COUNT(*) AS num_exceptions - FROM "LiteLLM_ErrorLogs" - WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp - GROUP BY model_group, api_base, exception_type - ORDER BY num_exceptions DESC + WITH cte AS ( + SELECT + CASE WHEN api_base = '' THEN litellm_model_name ELSE CONCAT(litellm_model_name, '-', api_base) END AS combined_model_api_base, + exception_type, + COUNT(*) AS num_exceptions + FROM "LiteLLM_ErrorLogs" + WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp + GROUP BY combined_model_api_base, exception_type + ) + SELECT + combined_model_api_base, + COUNT(*) AS total_exceptions, + json_object_agg(exception_type, num_exceptions) AS exception_counts + FROM cte + GROUP BY combined_model_api_base + ORDER BY total_exceptions DESC LIMIT 200; """ db_response = await prisma_client.db.query_raw(sql_query, startTime, endTime) response: List[dict] = [] exception_types = set() - for model_data in db_response: - model = model_data.get("model_group", "") - api_base = model_data.get("api_base", "") - exception_type = model_data.get("exception_type", "") - num_exceptions = model_data.get("num_exceptions", 0) - exception_types.add(exception_type) - response.append( - { - "model_group": model + "-" + api_base, - exception_type: num_exceptions, - "num_exceptions": num_exceptions, - } - ) - - # sort all entries in descending order based on num_exceptions - response = sorted(response, key=lambda x: x["num_exceptions"], reverse=True) - - return { - "data": response, - "exception_types": list(exception_types), + """ + Return Data + { + "combined_model_api_base": "gpt-3.5-turbo-https://api.openai.com/v1/, + "total_exceptions": 5, + "BadRequestException": 5, + "TimeoutException": 2 } + """ + + if db_response is not None: + # loop through all models + for model_data in db_response: + model = model_data.get("combined_model_api_base", "") + total_exceptions = model_data.get("total_exceptions", 0) + exception_counts = model_data.get("exception_counts", {}) + curr_row = { + "model": model, + "total_exceptions": total_exceptions, + } + curr_row.update(exception_counts) + response.append(curr_row) + for k, v in exception_counts.items(): + exception_types.add(k) + + return {"data": response, "exception_types": list(exception_types)} @router.get( From 5da931f2977e9770984273cf31b64a0a2a25fcaf Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 18:48:32 -0700 Subject: [PATCH 12/14] ui - clean up table --- ui/litellm-dashboard/src/components/model_dashboard.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index cb2246bad24..215af6105d3 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -1065,7 +1065,7 @@ const handleEditSubmit = async (formValues: Record) => { - + {/*
Model @@ -1080,7 +1080,7 @@ const handleEditSubmit = async (formValues: Record) => { ))} -
+ */} From b9238a00af1e1b3b8252495c839603b93966d519 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 22:44:28 -0700 Subject: [PATCH 13/14] ui - show tokens / sec --- litellm/proxy/proxy_server.py | 109 +++++++++++++++++----------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3bc9e7d52c9..0dd4fbb43f2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7524,7 +7524,7 @@ async def model_info_v2( ) async def model_metrics( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), - _selected_model_group: Optional[str] = None, + _selected_model_group: Optional[str] = "gpt-4-32k", startTime: Optional[datetime] = None, endTime: Optional[datetime] = None, ): @@ -7538,61 +7538,62 @@ async def model_metrics( ) startTime = startTime or datetime.now() - timedelta(days=30) endTime = endTime or datetime.now() - if _selected_model_group and llm_router is not None: - _model_list = llm_router.get_model_list() - _relevant_api_bases = [] - for model in _model_list: - if model["model_name"] == _selected_model_group: - _litellm_params = model["litellm_params"] - _api_base = _litellm_params.get("api_base", "") - _relevant_api_bases.append(_api_base) - _relevant_api_bases.append(_api_base + "/openai/") - sql_query = """ - SELECT - CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base, - COUNT(*) AS num_requests, - AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) AS avg_latency_seconds - FROM "LiteLLM_SpendLogs" - WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp - AND api_base = ANY($3) - GROUP BY CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END - ORDER BY num_requests DESC - LIMIT 50; - """ - - db_response = await prisma_client.db.query_raw( - sql_query, startTime, endTime, _relevant_api_bases - ) - else: - - sql_query = """ - SELECT - CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base, - COUNT(*) AS num_requests, - AVG(EXTRACT(epoch FROM ("endTime" - "startTime")) / total_tokens) AS avg_latency_per_token - FROM "LiteLLM_SpendLogs" - WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp - GROUP BY CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END - ORDER BY num_requests DESC; - """ - - db_response = await prisma_client.db.query_raw(sql_query, startTime, endTime) - response: List[dict] = [] - if response is not None: - # loop through all models + sql_query = """ + SELECT api_base, model, DATE_TRUNC('day', "startTime")::DATE AS day, + CASE + WHEN SUM(total_tokens) = 0 THEN 0 + ELSE AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) / SUM(total_tokens) + END AS avg_latency_per_token + FROM "LiteLLM_SpendLogs" + WHERE "startTime" >= NOW() - INTERVAL '30 days' AND "model" = $1 + GROUP BY api_base, model, day + ORDER BY avg_latency_per_token DESC; + """ + _all_api_bases = set() + db_response = await prisma_client.db.query_raw( + sql_query, _selected_model_group, startTime, endTime + ) + _daily_entries: dict = {} # {"Jun 23": {"model1": 0.002, "model2": 0.003}} + if db_response is not None: for model_data in db_response: - model = model_data.get("combined_model_api_base", "") - num_requests = model_data.get("num_requests", 0) - avg_latency_per_token = model_data.get("avg_latency_per_token", 0) - response.append( - { - "model": model, - "num_requests": num_requests, - "avg_latency_per_token": avg_latency_per_token, - } - ) - return response + _api_base = model_data["api_base"] + _model = model_data["model"] + _day = model_data["day"] + _avg_latency_per_token = model_data["avg_latency_per_token"] + if _day not in _daily_entries: + _daily_entries[_day] = {} + _combined_model_name = str(_model) + if "https://" in _api_base: + _combined_model_name = str(_api_base) + + _all_api_bases.add(_combined_model_name) + _daily_entries[_day][_combined_model_name] = _avg_latency_per_token + + """ + each entry needs to be like this: + { + date: 'Jun 23', + 'gpt-4-https://api.openai.com/v1/': 0.002, + 'gpt-43-https://api.openai.com-12/v1/': 0.002, + } + """ + # convert daily entries to list of dicts + + response: List[dict] = [] + + # sort daily entries by date + _daily_entries = dict(sorted(_daily_entries.items(), key=lambda item: item[0])) + for day in _daily_entries: + entry = {"date": str(day)} + for model_key, latency in _daily_entries[day].items(): + entry[model_key] = latency.__round__(5) + response.append(entry) + + return { + "data": response, + "all_api_bases": list(_all_api_bases), + } @router.get( From 9bf99df7e2cc079b606a3f21f22b3ca288191599 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Apr 2024 22:51:25 -0700 Subject: [PATCH 14/14] ui - model analytics --- .../src/components/model_dashboard.tsx | 113 ++++++++++-------- 1 file changed, 64 insertions(+), 49 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 215af6105d3..40bcdf67529 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -19,7 +19,7 @@ import { import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react"; import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react"; import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall } from "./networking"; -import { BarChart } from "@tremor/react"; +import { BarChart, AreaChart } from "@tremor/react"; import { Button as Button2, Modal, @@ -192,7 +192,6 @@ const ModelDashboard: React.FC = ({ const [providerModels, setProviderModels] = useState>([]); // Explicitly typing providerModels as a string array const providers = Object.values(Providers).filter(key => isNaN(Number(key))); - const [selectedProvider, setSelectedProvider] = useState("OpenAI"); const [healthCheckResponse, setHealthCheckResponse] = useState(''); @@ -202,6 +201,7 @@ const ModelDashboard: React.FC = ({ const [selectedModelGroup, setSelectedModelGroup] = useState(null); const [modelLatencyMetrics, setModelLatencyMetrics] = useState([]); const [modelMetrics, setModelMetrics] = useState([]); + const [modelMetricsCategories, setModelMetricsCategories] = useState([]); const [modelExceptions, setModelExceptions] = useState([]); const [allExceptions, setAllExceptions] = useState([]); const [failureTableData, setFailureTableData] = useState([]); @@ -459,11 +459,11 @@ const handleEditSubmit = async (formValues: Record) => { console.log("Model metrics response:", modelMetricsResponse); // Sort by latency (avg_latency_per_token) - const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_per_token - a.avg_latency_per_token); - console.log("Sorted by latency:", sortedByLatency); - setModelMetrics(modelMetricsResponse); - setModelLatencyMetrics(sortedByLatency); + + setModelMetrics(modelMetricsResponse.data); + setModelMetricsCategories(modelMetricsResponse.all_api_bases); + const modelExceptionsResponse = await modelExceptionsCall( accessToken, @@ -475,38 +475,38 @@ const handleEditSubmit = async (formValues: Record) => { setModelExceptions(modelExceptionsResponse.data); setAllExceptions(modelExceptionsResponse.exception_types); - let successdeploymentToSuccess: Record = {}; - for (let i = 0; i < modelMetricsResponse.length; i++) { - let element = modelMetricsResponse[i]; - let _model_name = element.model; - let _num_requests = element.num_requests; - successdeploymentToSuccess[_model_name] = _num_requests - } - console.log("successdeploymentToSuccess:", successdeploymentToSuccess) + // let successdeploymentToSuccess: Record = {}; + // for (let i = 0; i < modelMetricsResponse.length; i++) { + // let element = modelMetricsResponse[i]; + // let _model_name = element.model; + // let _num_requests = element.num_requests; + // successdeploymentToSuccess[_model_name] = _num_requests + // } + // console.log("successdeploymentToSuccess:", successdeploymentToSuccess) - let failureTableData = []; - let _failureData = modelExceptionsResponse.data; - for (let i = 0; i < _failureData.length; i++) { - const model = _failureData[i]; - let _model_name = model.model; - let total_exceptions = model.total_exceptions; - let total_Requests = successdeploymentToSuccess[_model_name]; - if (total_Requests == null) { - total_Requests = 0 - } - let _data = { - model: _model_name, - total_exceptions: total_exceptions, - total_Requests: total_Requests, - failure_rate: total_Requests / total_exceptions - } - failureTableData.push(_data); - // sort failureTableData by failure_rate - failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); + // let failureTableData = []; + // let _failureData = modelExceptionsResponse.data; + // for (let i = 0; i < _failureData.length; i++) { + // const model = _failureData[i]; + // let _model_name = model.model; + // let total_exceptions = model.total_exceptions; + // let total_Requests = successdeploymentToSuccess[_model_name]; + // if (total_Requests == null) { + // total_Requests = 0 + // } + // let _data = { + // model: _model_name, + // total_exceptions: total_exceptions, + // total_Requests: total_Requests, + // failure_rate: total_Requests / total_exceptions + // } + // failureTableData.push(_data); + // // sort failureTableData by failure_rate + // failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); - setFailureTableData(failureTableData); - console.log("failureTableData:", failureTableData); - } + // setFailureTableData(failureTableData); + // console.log("failureTableData:", failureTableData); + // } } catch (error) { console.error("There was an error fetching the model data", error); @@ -673,11 +673,19 @@ const handleEditSubmit = async (formValues: Record) => { console.log("Model metrics response:", modelMetricsResponse); // Assuming modelMetricsResponse now contains the metric data for the specified model group - const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_seconds - a.avg_latency_seconds); - console.log("Sorted by latency:", sortedByLatency); - - setModelMetrics(modelMetricsResponse); - setModelLatencyMetrics(sortedByLatency); + setModelMetrics(modelMetricsResponse.data); + setModelMetricsCategories(modelMetricsResponse.all_api_bases); + + const modelExceptionsResponse = await modelExceptionsCall( + accessToken, + userID, + userRole, + modelGroup + ) + console.log("Model exceptions response:", modelExceptionsResponse); + setModelExceptions(modelExceptionsResponse.data); + setAllExceptions(modelExceptionsResponse.exception_types); + } catch (error) { console.error("Failed to fetch model metrics", error); } @@ -1039,9 +1047,6 @@ const handleEditSubmit = async (formValues: Record) => {

View how requests were load balanced within a model group

-

(Beta feature) only supported for Azure Model Groups

- - + - {/* + { modelMetrics && modelMetricsCategories && ( + + )} + + +
Model @@ -1080,7 +1098,7 @@ const handleEditSubmit = async (formValues: Record) => { ))} -
*/} + @@ -1123,10 +1141,7 @@ const handleEditSubmit = async (formValues: Record) => { colors={['indigo-300', 'rose-200', '#ffcc33']} yAxisWidth={30} /> - - -