diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c910664f15e..b7b2d0ab6b5 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -916,6 +916,7 @@ class LiteLLM_ErrorLogs(LiteLLMBase): request_id: Optional[str] = str(uuid.uuid4()) api_base: Optional[str] = "" model_group: Optional[str] = "" + litellm_model_name: Optional[str] = "" model_id: Optional[str] = "" request_kwargs: Optional[dict] = {} exception_type: Optional[str] = "" diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3a7821d2727..0dd4fbb43f2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1258,6 +1258,7 @@ async def _PROXY_failure_handler( request_id=str(uuid.uuid4()), model_group=_model_group, model_id=_model_id, + litellm_model_name=kwargs.get("model"), request_kwargs=_optional_params, api_base=api_base, exception_type=_exception_type, @@ -7523,9 +7524,9 @@ async def model_info_v2( ) async def model_metrics( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), - _selected_model_group: Optional[str] = None, - startTime: Optional[datetime] = datetime.now() - timedelta(days=30), - endTime: Optional[datetime] = datetime.now(), + _selected_model_group: Optional[str] = "gpt-4-32k", + startTime: Optional[datetime] = None, + endTime: Optional[datetime] = None, ): global prisma_client, llm_router if prisma_client is None: @@ -7535,65 +7536,142 @@ async def model_metrics( param="None", code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - if _selected_model_group and llm_router is not None: - _model_list = llm_router.get_model_list() - _relevant_api_bases = [] - for model in _model_list: - if model["model_name"] == _selected_model_group: - _litellm_params = model["litellm_params"] - _api_base = _litellm_params.get("api_base", "") - _relevant_api_bases.append(_api_base) - _relevant_api_bases.append(_api_base + "/openai/") + startTime = startTime or datetime.now() - timedelta(days=30) + endTime = endTime or datetime.now() + + sql_query = """ + SELECT api_base, model, DATE_TRUNC('day', "startTime")::DATE AS day, + CASE + WHEN SUM(total_tokens) = 0 THEN 0 + ELSE AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) / SUM(total_tokens) + END AS avg_latency_per_token + FROM "LiteLLM_SpendLogs" + WHERE "startTime" >= NOW() - INTERVAL '30 days' AND "model" = $1 + GROUP BY api_base, model, day + ORDER BY avg_latency_per_token DESC; + """ + _all_api_bases = set() + db_response = await prisma_client.db.query_raw( + sql_query, _selected_model_group, startTime, endTime + ) + _daily_entries: dict = {} # {"Jun 23": {"model1": 0.002, "model2": 0.003}} + if db_response is not None: + for model_data in db_response: + _api_base = model_data["api_base"] + _model = model_data["model"] + _day = model_data["day"] + _avg_latency_per_token = model_data["avg_latency_per_token"] + if _day not in _daily_entries: + _daily_entries[_day] = {} + _combined_model_name = str(_model) + if "https://" in _api_base: + _combined_model_name = str(_api_base) + + _all_api_bases.add(_combined_model_name) + _daily_entries[_day][_combined_model_name] = _avg_latency_per_token - sql_query = """ - SELECT - CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base, - COUNT(*) AS num_requests, - AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) AS avg_latency_seconds - FROM "LiteLLM_SpendLogs" - WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp - AND api_base = ANY($3) - GROUP BY CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END - ORDER BY num_requests DESC - LIMIT 50; """ + each entry needs to be like this: + { + date: 'Jun 23', + 'gpt-4-https://api.openai.com/v1/': 0.002, + 'gpt-43-https://api.openai.com-12/v1/': 0.002, + } + """ + # convert daily entries to list of dicts - db_response = await prisma_client.db.query_raw( - sql_query, startTime, endTime, _relevant_api_bases + response: List[dict] = [] + + # sort daily entries by date + _daily_entries = dict(sorted(_daily_entries.items(), key=lambda item: item[0])) + for day in _daily_entries: + entry = {"date": str(day)} + for model_key, latency in _daily_entries[day].items(): + entry[model_key] = latency.__round__(5) + response.append(entry) + + return { + "data": response, + "all_api_bases": list(_all_api_bases), + } + + +@router.get( + "/model/metrics/exceptions", + description="View number of failed requests per model on config.yaml", + tags=["model management"], + include_in_schema=False, + dependencies=[Depends(user_api_key_auth)], +) +async def model_metrics_exceptions( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + _selected_model_group: Optional[str] = None, + startTime: Optional[datetime] = None, + endTime: Optional[datetime] = None, +): + global prisma_client, llm_router + if prisma_client is None: + raise ProxyException( + message="Prisma Client is not initialized", + type="internal_error", + param="None", + code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - else: - sql_query = """ - SELECT - CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END AS combined_model_api_base, - COUNT(*) AS num_requests, - AVG(EXTRACT(epoch FROM ("endTime" - "startTime"))) AS avg_latency_seconds - FROM - "LiteLLM_SpendLogs" + startTime = startTime or datetime.now() - timedelta(days=30) + endTime = endTime or datetime.now() + + """ + """ + sql_query = """ + WITH cte AS ( + SELECT + CASE WHEN api_base = '' THEN litellm_model_name ELSE CONCAT(litellm_model_name, '-', api_base) END AS combined_model_api_base, + exception_type, + COUNT(*) AS num_exceptions + FROM "LiteLLM_ErrorLogs" WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp - GROUP BY - CASE WHEN api_base = '' THEN model ELSE CONCAT(model, '-', api_base) END - ORDER BY - num_requests DESC - LIMIT 50; - """ - - db_response = await prisma_client.db.query_raw(sql_query, startTime, endTime) + GROUP BY combined_model_api_base, exception_type + ) + SELECT + combined_model_api_base, + COUNT(*) AS total_exceptions, + json_object_agg(exception_type, num_exceptions) AS exception_counts + FROM cte + GROUP BY combined_model_api_base + ORDER BY total_exceptions DESC + LIMIT 200; + """ + db_response = await prisma_client.db.query_raw(sql_query, startTime, endTime) response: List[dict] = [] - if response is not None: + exception_types = set() + + """ + Return Data + { + "combined_model_api_base": "gpt-3.5-turbo-https://api.openai.com/v1/, + "total_exceptions": 5, + "BadRequestException": 5, + "TimeoutException": 2 + } + """ + + if db_response is not None: # loop through all models for model_data in db_response: model = model_data.get("combined_model_api_base", "") - num_requests = model_data.get("num_requests", 0) - avg_latency_seconds = model_data.get("avg_latency_seconds", 0) - response.append( - { - "model": model, - "num_requests": num_requests, - "avg_latency_seconds": avg_latency_seconds, - } - ) - return response + total_exceptions = model_data.get("total_exceptions", 0) + exception_counts = model_data.get("exception_counts", {}) + curr_row = { + "model": model, + "total_exceptions": total_exceptions, + } + curr_row.update(exception_counts) + response.append(curr_row) + for k, v in exception_counts.items(): + exception_types.add(k) + + return {"data": response, "exception_types": list(exception_types)} @router.get( diff --git a/schema.prisma b/schema.prisma index b362a0ec025..f078dbcf432 100644 --- a/schema.prisma +++ b/schema.prisma @@ -190,6 +190,7 @@ model LiteLLM_ErrorLogs { endTime DateTime // Assuming end_time is a DateTime field api_base String @default("") model_group String @default("") // public model_name / model_group + litellm_model_name String @default("") // model passed to litellm model_id String @default("") // ID of model in ProxyModelTable request_kwargs Json @default("{}") exception_type String @default("") diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 2491b40e627..40bcdf67529 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -18,8 +18,8 @@ import { } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon } from "@tremor/react"; import { Select, SelectItem, MultiSelect, MultiSelectItem } from "@tremor/react"; -import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall } from "./networking"; -import { BarChart } from "@tremor/react"; +import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall } from "./networking"; +import { BarChart, AreaChart } from "@tremor/react"; import { Button as Button2, Modal, @@ -192,7 +192,6 @@ const ModelDashboard: React.FC = ({ const [providerModels, setProviderModels] = useState>([]); // Explicitly typing providerModels as a string array const providers = Object.values(Providers).filter(key => isNaN(Number(key))); - const [selectedProvider, setSelectedProvider] = useState("OpenAI"); const [healthCheckResponse, setHealthCheckResponse] = useState(''); @@ -200,6 +199,12 @@ const ModelDashboard: React.FC = ({ const [selectedModel, setSelectedModel] = useState(null); const [availableModelGroups, setAvailableModelGroups] = useState>([]); const [selectedModelGroup, setSelectedModelGroup] = useState(null); + const [modelLatencyMetrics, setModelLatencyMetrics] = useState([]); + const [modelMetrics, setModelMetrics] = useState([]); + const [modelMetricsCategories, setModelMetricsCategories] = useState([]); + const [modelExceptions, setModelExceptions] = useState([]); + const [allExceptions, setAllExceptions] = useState([]); + const [failureTableData, setFailureTableData] = useState([]); const EditModelModal: React.FC = ({ visible, onCancel, model, onSubmit }) => { const [form] = Form.useForm(); @@ -445,12 +450,64 @@ const handleEditSubmit = async (formValues: Record) => { let _array_model_groups = Array.from(all_model_groups) setAvailableModelGroups(_array_model_groups); - // if userRole is Admin, show the pending requests - if (userRole === "Admin" && accessToken) { - const user_requests = await userGetRequesedtModelsCall(accessToken); - console.log("Pending Requests:", pendingRequests); - setPendingRequests(user_requests.requests || []); - } + const modelMetricsResponse = await modelMetricsCall( + accessToken, + userID, + userRole, + null + ); + + console.log("Model metrics response:", modelMetricsResponse); + // Sort by latency (avg_latency_per_token) + + + setModelMetrics(modelMetricsResponse.data); + setModelMetricsCategories(modelMetricsResponse.all_api_bases); + + + const modelExceptionsResponse = await modelExceptionsCall( + accessToken, + userID, + userRole, + null + ) + console.log("Model exceptions response:", modelExceptionsResponse); + setModelExceptions(modelExceptionsResponse.data); + setAllExceptions(modelExceptionsResponse.exception_types); + + // let successdeploymentToSuccess: Record = {}; + // for (let i = 0; i < modelMetricsResponse.length; i++) { + // let element = modelMetricsResponse[i]; + // let _model_name = element.model; + // let _num_requests = element.num_requests; + // successdeploymentToSuccess[_model_name] = _num_requests + // } + // console.log("successdeploymentToSuccess:", successdeploymentToSuccess) + + // let failureTableData = []; + // let _failureData = modelExceptionsResponse.data; + // for (let i = 0; i < _failureData.length; i++) { + // const model = _failureData[i]; + // let _model_name = model.model; + // let total_exceptions = model.total_exceptions; + // let total_Requests = successdeploymentToSuccess[_model_name]; + // if (total_Requests == null) { + // total_Requests = 0 + // } + // let _data = { + // model: _model_name, + // total_exceptions: total_exceptions, + // total_Requests: total_Requests, + // failure_rate: total_Requests / total_exceptions + // } + // failureTableData.push(_data); + // // sort failureTableData by failure_rate + // failureTableData.sort((a, b) => b.failure_rate - a.failure_rate); + + // setFailureTableData(failureTableData); + // console.log("failureTableData:", failureTableData); + // } + } catch (error) { console.error("There was an error fetching the model data", error); } @@ -603,6 +660,38 @@ const handleEditSubmit = async (formValues: Record) => { }; + const updateModelMetrics = async (modelGroup: string | null) => { + console.log("Updating model metrics for group:", modelGroup); + if (!accessToken || !userID || !userRole) { + return + } + setSelectedModelGroup(modelGroup); // If you want to store the selected model group in state + + + try { + const modelMetricsResponse = await modelMetricsCall(accessToken, userID, userRole, modelGroup); + console.log("Model metrics response:", modelMetricsResponse); + + // Assuming modelMetricsResponse now contains the metric data for the specified model group + setModelMetrics(modelMetricsResponse.data); + setModelMetricsCategories(modelMetricsResponse.all_api_bases); + + const modelExceptionsResponse = await modelExceptionsCall( + accessToken, + userID, + userRole, + modelGroup + ) + console.log("Model exceptions response:", modelExceptionsResponse); + setModelExceptions(modelExceptionsResponse.data); + setAllExceptions(modelExceptionsResponse.exception_types); + + } catch (error) { + console.error("Failed to fetch model metrics", error); + } + } + + const getPlaceholder = (selectedProvider: string): string => { if (selectedProvider === Providers.Vertex_AI) { @@ -640,6 +729,7 @@ const handleEditSubmit = async (formValues: Record) => { All Models Add Model 
/health Models
 + Model Analytics
@@ -955,6 +1045,105 @@ const handleEditSubmit = async (formValues: Record) => { + +

View how requests were load balanced within a model group

+ + + + + + { modelMetrics && modelMetricsCategories && ( + + )} + + + + + + Model + Median Latency/Token + + + + {modelLatencyMetrics.map((metric, idx) => ( + + {metric.model} + {metric.avg_latency_per_token.toFixed(4)} + + ))} + +
+ + + + + + + + Model + Success Requests + Error Requests + Failure % + + + + + {failureTableData.map((metric, idx) => ( + + {metric.model} + {metric.total_Requests} + {metric.total_exceptions} + {metric.failure_rate}% + + + ))} + +
+ + + + + + + Exceptions per Model + + + + diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index db03a2bd000..ed17b7daa63 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -474,6 +474,43 @@ export const modelMetricsCall = async ( } }; + +export const modelExceptionsCall = async ( + accessToken: String, + userID: String, + userRole: String, + modelGroup: String | null, +) => { + /** + * Get all models on proxy + */ + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/model/metrics/exceptions` : `/model/metrics/exceptions`; + + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData, 20); + throw new Error("Network response was not ok"); + } + const data = await response.json(); + // message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + + export const modelAvailableCall = async ( accessToken: String, userID: String, diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/components/usage.tsx index 0caeaccc468..f0b4f0551a7 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/components/usage.tsx @@ -146,10 +146,6 @@ const UsagePage: React.FC = ({ const [topTagsData, setTopTagsData] = useState([]); const [uniqueTeamIds, setUniqueTeamIds] = useState([]); const [totalSpendPerTeam, setTotalSpendPerTeam] = useState([]); - const [modelMetrics, setModelMetrics] = useState([]); - const [modelLatencyMetrics, setModelLatencyMetrics] = useState([]); - const [modelGroups, setModelGroups] = useState([]); - const [selectedModelGroup, setSelectedModelGroup] = useState(null); const firstDay = new Date( currentDate.getFullYear(), @@ -231,25 +227,6 @@ const UsagePage: React.FC = ({ const top_tags = await tagsSpendLogsCall(accessToken); setTopTagsData(top_tags.top_10_tags); - // get model groups - const _model_groups = await modelInfoCall(accessToken, userID, userRole); - let model_groups = _model_groups.data; - console.log("model groups in model dashboard", model_groups); - - let available_model_groups = []; - // loop through each model in model_group, access litellm_params and only inlclude the model if model["litellm_params"]["model"] startswith "azure/" - for (let i = 0; i < model_groups.length; i++) { - let model = model_groups[i]; - console.log("model check", model); - let model_group = model["litellm_params"]["model"]; - console.log("model group", model_group); - if (model_group.startsWith("azure/")) { - available_model_groups.push(model["model_name"]); - } - } - setModelGroups(available_model_groups); - - } else if (userRole == "App Owner") { await userSpendLogsCall( accessToken, @@ -286,22 +263,6 @@ const UsagePage: React.FC = ({ } }); } - - const modelMetricsResponse = await modelMetricsCall( - accessToken, - userID, - userRole, - null - ); - - console.log("Model metrics response:", modelMetricsResponse); - // Sort by latency (avg_latency_seconds) - const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_seconds - a.avg_latency_seconds); - console.log("Sorted by latency:", sortedByLatency); - - setModelMetrics(modelMetricsResponse); - setModelLatencyMetrics(sortedByLatency); - } catch (error) { console.error("There was an error fetching the data", error); // Optionally, update your UI to reflect the error state here as well @@ -312,30 +273,6 @@ const UsagePage: React.FC = ({ }, [accessToken, token, userRole, userID, startTime, endTime]); - const updateModelMetrics = async (modelGroup: string | null) => { - console.log("Updating model metrics for group:", modelGroup); - if (!accessToken || !userID || !userRole) { - return - } - setSelectedModelGroup(modelGroup); // If you want to store the selected model group in state - - - try { - const modelMetricsResponse = await modelMetricsCall(accessToken, userID, userRole, modelGroup); - console.log("Model metrics response:", modelMetricsResponse); - - // Assuming modelMetricsResponse now contains the metric data for the specified model group - const sortedByLatency = [...modelMetricsResponse].sort((a, b) => b.avg_latency_seconds - a.avg_latency_seconds); - console.log("Sorted by latency:", sortedByLatency); - - setModelMetrics(modelMetricsResponse); - setModelLatencyMetrics(sortedByLatency); - } catch (error) { - console.error("Failed to fetch model metrics", error); - } - } - - return (
= ({ All Up Team Based Usage Tag Based Usage - Model Based Usage @@ -492,60 +428,6 @@ const UsagePage: React.FC = ({ - - Filter By Model Group -

View how requests were load balanced within a model group

-

(Beta feature) only supported for Azure Model Groups

- - - - - Number Requests per Model - - - - Latency Per Model - - - -