mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
Merge pull request #4340 from BerriAI/litellm_show_cache_hit_stats
[Feat] Admin UI - Show Cache hit stats
This commit is contained in:
Ishaan Jaff
GitHub
commit
cc11cedf02
6 changed files with 496 additions and 7 deletions
109
litellm/proxy/analytics_endpoints/analytics_endpoints.py
Normal file
109
litellm/proxy/analytics_endpoints/analytics_endpoints.py
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
#### Analytics Endpoints #####
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import List, Optional
|
||||
|
||||
import fastapi
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get(
|
||||
"/global/activity/cache_hits",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
responses={
|
||||
200: {"model": List[LiteLLM_SpendLogs]},
|
||||
},
|
||||
include_in_schema=False,
|
||||
)
|
||||
async def get_global_activity(
|
||||
start_date: Optional[str] = fastapi.Query(
|
||||
default=None,
|
||||
description="Time from which to start viewing spend",
|
||||
),
|
||||
end_date: Optional[str] = fastapi.Query(
|
||||
default=None,
|
||||
description="Time till which to view spend",
|
||||
),
|
||||
):
|
||||
"""
|
||||
Get number of cache hits, vs misses
|
||||
|
||||
{
|
||||
"daily_data": [
|
||||
const chartdata = [
|
||||
{
|
||||
date: 'Jan 22',
|
||||
cache_hits: 10,
|
||||
llm_api_calls: 2000
|
||||
},
|
||||
{
|
||||
date: 'Jan 23',
|
||||
cache_hits: 10,
|
||||
llm_api_calls: 12
|
||||
},
|
||||
],
|
||||
"sum_cache_hits": 20,
|
||||
"sum_llm_api_calls": 2012
|
||||
}
|
||||
"""
|
||||
from collections import defaultdict
|
||||
|
||||
if start_date is None or end_date is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={"error": "Please provide start_date and end_date"},
|
||||
)
|
||||
|
||||
start_date_obj = datetime.strptime(start_date, "%Y-%m-%d")
|
||||
end_date_obj = datetime.strptime(end_date, "%Y-%m-%d")
|
||||
|
||||
from litellm.proxy.proxy_server import llm_router, prisma_client
|
||||
|
||||
try:
|
||||
if prisma_client is None:
|
||||
raise ValueError(
|
||||
f"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys"
|
||||
)
|
||||
|
||||
sql_query = """
|
||||
SELECT
|
||||
CASE
|
||||
WHEN vt."key_alias" IS NOT NULL THEN vt."key_alias"
|
||||
ELSE 'Unnamed Key'
|
||||
END AS api_key,
|
||||
sl."call_type",
|
||||
sl."model",
|
||||
COUNT(*) AS total_rows,
|
||||
SUM(CASE WHEN sl."cache_hit" = 'True' THEN 1 ELSE 0 END) AS cache_hit_true_rows,
|
||||
SUM(CASE WHEN sl."cache_hit" = 'True' THEN sl."completion_tokens" ELSE 0 END) AS cached_completion_tokens,
|
||||
SUM(CASE WHEN sl."cache_hit" != 'True' THEN sl."completion_tokens" ELSE 0 END) AS generated_completion_tokens
|
||||
FROM "LiteLLM_SpendLogs" sl
|
||||
LEFT JOIN "LiteLLM_VerificationToken" vt ON sl."api_key" = vt."token"
|
||||
WHERE
|
||||
sl."startTime" BETWEEN $1::date AND $2::date + interval '1 day'
|
||||
GROUP BY
|
||||
vt."key_alias",
|
||||
sl."call_type",
|
||||
sl."model"
|
||||
"""
|
||||
db_response = await prisma_client.db.query_raw(
|
||||
sql_query, start_date_obj, end_date_obj
|
||||
)
|
||||
|
||||
if db_response is None:
|
||||
return []
|
||||
|
||||
return db_response
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={"error": str(e)},
|
||||
)
|
||||
|
|
@ -116,6 +116,9 @@ from litellm.exceptions import RejectedRequestError
|
|||
from litellm.integrations.slack_alerting import SlackAlerting, SlackAlertingArgs
|
||||
from litellm.llms.custom_httpx.httpx_handler import HTTPHandler
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.analytics_endpoints.analytics_endpoints import (
|
||||
router as analytics_router,
|
||||
)
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
allowed_routes_check,
|
||||
common_checks,
|
||||
|
|
@ -9139,3 +9142,4 @@ app.include_router(internal_user_router)
|
|||
app.include_router(team_router)
|
||||
app.include_router(spend_management_router)
|
||||
app.include_router(caching_router)
|
||||
app.include_router(analytics_router)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import APIRef from "@/components/api_ref";
|
|||
import ChatUI from "@/components/chat_ui";
|
||||
import Sidebar from "../components/leftnav";
|
||||
import Usage from "../components/usage";
|
||||
import CacheDashboard from "@/components/cache_dashboard";
|
||||
import { jwtDecode } from "jwt-decode";
|
||||
import { Typography } from "antd";
|
||||
|
||||
|
|
@ -221,6 +222,14 @@ const CreateKeyPage = () => {
|
|||
publicPage={false}
|
||||
premiumUser={premiumUser}
|
||||
/>
|
||||
) : page == "caching" ? (
|
||||
<CacheDashboard
|
||||
userID={userID}
|
||||
userRole={userRole}
|
||||
token={token}
|
||||
accessToken={accessToken}
|
||||
premiumUser={premiumUser}
|
||||
/>
|
||||
) : (
|
||||
<Usage
|
||||
userID={userID}
|
||||
|
|
|
|||
322
ui/litellm-dashboard/src/components/cache_dashboard.tsx
Normal file
322
ui/litellm-dashboard/src/components/cache_dashboard.tsx
Normal file
|
|
@ -0,0 +1,322 @@
|
|||
import React, { useState, useEffect } from "react";
|
||||
import {
|
||||
Card,
|
||||
Title,
|
||||
BarChart,
|
||||
Subtitle,
|
||||
Grid,
|
||||
Col,
|
||||
Select,
|
||||
SelectItem,
|
||||
DateRangePicker,
|
||||
DateRangePickerValue,
|
||||
MultiSelect,
|
||||
MultiSelectItem,
|
||||
} from "@tremor/react";
|
||||
|
||||
import {
|
||||
adminGlobalCacheActivity,
|
||||
} from "./networking";
|
||||
|
||||
const formatDateWithoutTZ = (date: Date | undefined) => {
|
||||
if (!date) return undefined;
|
||||
return date.toISOString().split('T')[0];
|
||||
};
|
||||
|
||||
|
||||
function valueFormatterNumbers(number: number) {
|
||||
const formatter = new Intl.NumberFormat('en-US', {
|
||||
maximumFractionDigits: 0,
|
||||
notation: 'compact',
|
||||
compactDisplay: 'short',
|
||||
});
|
||||
|
||||
return formatter.format(number);
|
||||
}
|
||||
|
||||
interface CachePageProps {
|
||||
accessToken: string | null;
|
||||
token: string | null;
|
||||
userRole: string | null;
|
||||
userID: string | null;
|
||||
premiumUser: boolean;
|
||||
}
|
||||
|
||||
interface cacheDataItem {
|
||||
api_key: string;
|
||||
model: string;
|
||||
cache_hit_true_rows: number;
|
||||
cached_completion_tokens: number;
|
||||
total_rows: number;
|
||||
generated_completion_tokens: number;
|
||||
call_type: string;
|
||||
|
||||
// Add other properties as needed
|
||||
}
|
||||
|
||||
|
||||
interface uiData {
|
||||
"name": string;
|
||||
"LLM API requests": number;
|
||||
"Cache hit": number;
|
||||
"Cached Completion Tokens": number;
|
||||
"Generated Completion Tokens": number;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
const CacheDashboard: React.FC<CachePageProps> = ({
|
||||
accessToken,
|
||||
token,
|
||||
userRole,
|
||||
userID,
|
||||
premiumUser,
|
||||
}) => {
|
||||
const [filteredData, setFilteredData] = useState<uiData[]>([]);
|
||||
const [selectedApiKeys, setSelectedApiKeys] = useState<string[]>([]);
|
||||
const [selectedModels, setSelectedModels] = useState<string[]>([]);
|
||||
const [data, setData] = useState<cacheDataItem[]>([]);
|
||||
const [cachedResponses, setCachedResponses] = useState("0");
|
||||
const [cachedTokens, setCachedTokens] = useState("0");
|
||||
const [cacheHitRatio, setCacheHitRatio] = useState("0");
|
||||
|
||||
const [dateValue, setDateValue] = useState<DateRangePickerValue>({
|
||||
from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000),
|
||||
to: new Date(),
|
||||
});
|
||||
|
||||
|
||||
useEffect(() => {
|
||||
if (!accessToken || !dateValue) {
|
||||
return;
|
||||
}
|
||||
const fetchData = async () => {
|
||||
const response = await adminGlobalCacheActivity(accessToken, formatDateWithoutTZ(dateValue.from), formatDateWithoutTZ(dateValue.to));
|
||||
setData(response);
|
||||
};
|
||||
fetchData();
|
||||
}, [accessToken]);
|
||||
|
||||
const uniqueApiKeys = Array.from(new Set(data.map((item) => item?.api_key ?? "")));
|
||||
const uniqueModels = Array.from(new Set(data.map((item) => item?.model ?? "")));
|
||||
const uniqueCallTypes = Array.from(new Set(data.map((item) => item?.call_type ?? "")));
|
||||
|
||||
|
||||
const updateCachingData = async (startTime: Date | undefined, endTime: Date | undefined) => {
|
||||
if (!startTime || !endTime || !accessToken) {
|
||||
return;
|
||||
}
|
||||
|
||||
// the endTime put it to the last hour of the selected date
|
||||
endTime.setHours(23, 59, 59, 999);
|
||||
|
||||
// startTime put it to the first hour of the selected date
|
||||
startTime.setHours(0, 0, 0, 0);
|
||||
|
||||
let new_cache_data = await adminGlobalCacheActivity(
|
||||
accessToken,
|
||||
formatDateWithoutTZ(startTime),
|
||||
formatDateWithoutTZ(endTime)
|
||||
)
|
||||
|
||||
setData(new_cache_data);
|
||||
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
console.log("DATA IN CACHE DASHBOARD", data);
|
||||
let newData: cacheDataItem[] = data;
|
||||
if (selectedApiKeys.length > 0) {
|
||||
newData = newData.filter((item) => selectedApiKeys.includes(item.api_key));
|
||||
}
|
||||
|
||||
if (selectedModels.length > 0) {
|
||||
newData = newData.filter((item) => selectedModels.includes(item.model));
|
||||
}
|
||||
|
||||
/*
|
||||
Data looks like this
|
||||
[{"api_key":"147dba2181f28914eea90eb484926c293cdcf7f5b5c9c3dd6a004d9e0f9fdb21","call_type":"acompletion","model":"llama3-8b-8192","total_rows":13,"cache_hit_true_rows":0},
|
||||
{"api_key":"8c23f021d0535c2e59abb7d83d0e03ccfb8db1b90e231ff082949d95df419e86","call_type":"None","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0},
|
||||
{"api_key":"88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b","call_type":"acompletion","model":"gpt-3.5-turbo","total_rows":19,"cache_hit_true_rows":0},
|
||||
{"api_key":"88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b","call_type":"aimage_generation","model":"","total_rows":3,"cache_hit_true_rows":0},
|
||||
{"api_key":"0ad4b3c03dcb6de0b5b8f761db798c6a8ae80be3fd1e2ea30c07ce6d5e3bf870","call_type":"None","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0},
|
||||
{"api_key":"034224b36e9769bc50e2190634abc3f97cad789b17ca80ac43b82f46cd5579b3","call_type":"","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0},
|
||||
{"api_key":"4f9c71cce0a2bb9a0b62ce6f0ebb3245b682702a8851d26932fa7e3b8ebfc755","call_type":"","model":"chatgpt-v-2","total_rows":1,"cache_hit_true_rows":0},
|
||||
*/
|
||||
|
||||
// What data we need for bar chat
|
||||
// ui_data = [
|
||||
// {
|
||||
// name: "Call Type",
|
||||
// Cache hit: 20,
|
||||
// LLM API requests: 10,
|
||||
// }
|
||||
// ]
|
||||
|
||||
console.log("before processed data in cache dashboard", newData);
|
||||
|
||||
let llm_api_requests = 0;
|
||||
let cache_hits = 0;
|
||||
let cached_tokens = 0;
|
||||
const processedData = newData.reduce((acc: uiData[], item) => {
|
||||
console.log("Processing item:", item);
|
||||
|
||||
if (!item.call_type) {
|
||||
console.log("Item has no call_type:", item);
|
||||
item.call_type = "Unknown";
|
||||
}
|
||||
|
||||
|
||||
llm_api_requests += (item.total_rows || 0) - (item.cache_hit_true_rows || 0);
|
||||
cache_hits += item.cache_hit_true_rows || 0;
|
||||
cached_tokens += item.cached_completion_tokens || 0;
|
||||
|
||||
const existingItem = acc.find(i => i.name === item.call_type);
|
||||
if (existingItem) {
|
||||
existingItem["LLM API requests"] += (item.total_rows || 0) - (item.cache_hit_true_rows || 0);
|
||||
existingItem["Cache hit"] += item.cache_hit_true_rows || 0;
|
||||
existingItem["Cached Completion Tokens"] += item.cached_completion_tokens || 0;
|
||||
existingItem["Generated Completion Tokens"] += item.generated_completion_tokens || 0;
|
||||
} else {
|
||||
acc.push({
|
||||
name: item.call_type,
|
||||
"LLM API requests": (item.total_rows || 0) - (item.cache_hit_true_rows || 0),
|
||||
"Cache hit": item.cache_hit_true_rows || 0,
|
||||
"Cached Completion Tokens": item.cached_completion_tokens || 0,
|
||||
"Generated Completion Tokens": item.generated_completion_tokens || 0
|
||||
});
|
||||
}
|
||||
return acc;
|
||||
}, []);
|
||||
|
||||
// set header cache statistics
|
||||
setCachedResponses(valueFormatterNumbers(cache_hits));
|
||||
setCachedTokens(valueFormatterNumbers(cached_tokens));
|
||||
if (llm_api_requests > 0) {
|
||||
let cache_hit_ratio = ((cache_hits / llm_api_requests) * 100).toFixed(2);
|
||||
setCacheHitRatio(cache_hit_ratio);
|
||||
} else {
|
||||
setCacheHitRatio("0");
|
||||
}
|
||||
|
||||
setFilteredData(processedData);
|
||||
|
||||
console.log("PROCESSED DATA IN CACHE DASHBOARD", processedData);
|
||||
|
||||
}, [selectedApiKeys, selectedModels, dateValue, data]);
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<Grid numItems={3} className="gap-4 mt-4">
|
||||
<Col>
|
||||
<MultiSelect
|
||||
placeholder="Select API Keys"
|
||||
value={selectedApiKeys}
|
||||
onValueChange={setSelectedApiKeys}
|
||||
>
|
||||
{uniqueApiKeys.map((key) => (
|
||||
<MultiSelectItem key={key} value={key}>
|
||||
{key}
|
||||
</MultiSelectItem>
|
||||
))}
|
||||
</MultiSelect>
|
||||
</Col>
|
||||
<Col>
|
||||
<MultiSelect
|
||||
placeholder="Select Models"
|
||||
value={selectedModels}
|
||||
onValueChange={setSelectedModels}
|
||||
>
|
||||
{uniqueModels.map((model) => (
|
||||
<MultiSelectItem key={model} value={model}>
|
||||
{model}
|
||||
</MultiSelectItem>
|
||||
))}
|
||||
</MultiSelect>
|
||||
</Col>
|
||||
<Col>
|
||||
<DateRangePicker
|
||||
enableSelect={true}
|
||||
value={dateValue}
|
||||
onValueChange={(value) => {
|
||||
setDateValue(value);
|
||||
updateCachingData(value.from, value.to);
|
||||
}}
|
||||
selectPlaceholder="Select date range"
|
||||
/>
|
||||
</Col>
|
||||
</Grid>
|
||||
|
||||
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-3 mt-4">
|
||||
<Card>
|
||||
<p className="text-tremor-default font-medium text-tremor-content dark:text-dark-tremor-content">
|
||||
Cache Hit Ratio
|
||||
</p>
|
||||
<div className="mt-2 flex items-baseline space-x-2.5">
|
||||
<p className="text-tremor-metric font-semibold text-tremor-content-strong dark:text-dark-tremor-content-strong">
|
||||
{cacheHitRatio}%
|
||||
</p>
|
||||
|
||||
</div>
|
||||
</Card>
|
||||
<Card>
|
||||
<p className="text-tremor-default font-medium text-tremor-content dark:text-dark-tremor-content">
|
||||
Cache Hits
|
||||
</p>
|
||||
<div className="mt-2 flex items-baseline space-x-2.5">
|
||||
<p className="text-tremor-metric font-semibold text-tremor-content-strong dark:text-dark-tremor-content-strong">
|
||||
{cachedResponses}
|
||||
</p>
|
||||
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<p className="text-tremor-default font-medium text-tremor-content dark:text-dark-tremor-content">
|
||||
Cached Tokens
|
||||
</p>
|
||||
<div className="mt-2 flex items-baseline space-x-2.5">
|
||||
<p className="text-tremor-metric font-semibold text-tremor-content-strong dark:text-dark-tremor-content-strong">
|
||||
{cachedTokens}
|
||||
</p>
|
||||
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
</div>
|
||||
|
||||
<Subtitle className="mt-4">Cache Hits vs API Requests</Subtitle>
|
||||
<BarChart
|
||||
title="Cache Hits vs API Requests"
|
||||
data={filteredData}
|
||||
index="name"
|
||||
valueFormatter={valueFormatterNumbers}
|
||||
categories={["LLM API requests", "Cache hit"]}
|
||||
colors={["sky", "teal"]}
|
||||
yAxisWidth={48}
|
||||
/>
|
||||
|
||||
<Subtitle className="mt-4">Cached Completion Tokens vs Generated Completion Tokens</Subtitle>
|
||||
<BarChart
|
||||
className="mt-6"
|
||||
data={filteredData}
|
||||
index="name"
|
||||
valueFormatter={valueFormatterNumbers}
|
||||
categories={["Generated Completion Tokens", "Cached Completion Tokens"]}
|
||||
colors={["sky", "teal"]}
|
||||
yAxisWidth={48}
|
||||
/>
|
||||
|
||||
|
||||
</Card>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
);
|
||||
};
|
||||
|
||||
export default CacheDashboard;
|
||||
|
|
@ -79,28 +79,32 @@ const Sidebar: React.FC<SidebarProps> = ({
|
|||
<Text>Logging & Alerts</Text>
|
||||
</Menu.Item>
|
||||
) : null}
|
||||
{userRole == "Admin" ? (
|
||||
<Menu.Item key="9" onClick={() => setPage("caching")}>
|
||||
<Text>Caching</Text>
|
||||
</Menu.Item>
|
||||
) : null}
|
||||
|
||||
{userRole == "Admin" ? (
|
||||
<Menu.Item key="9" onClick={() => setPage("budgets")}>
|
||||
<Menu.Item key="10" onClick={() => setPage("budgets")}>
|
||||
<Text>Budgets</Text>
|
||||
</Menu.Item>
|
||||
) : null}
|
||||
|
||||
|
||||
{userRole == "Admin" ? (
|
||||
<Menu.Item key="10" onClick={() => setPage("general-settings")}>
|
||||
<Menu.Item key="11" onClick={() => setPage("general-settings")}>
|
||||
<Text>Router Settings</Text>
|
||||
</Menu.Item>
|
||||
) : null}
|
||||
|
||||
{userRole == "Admin" ? (
|
||||
<Menu.Item key="11" onClick={() => setPage("admin-panel")}>
|
||||
<Menu.Item key="12" onClick={() => setPage("admin-panel")}>
|
||||
<Text>Admin</Text>
|
||||
</Menu.Item>
|
||||
) : null}
|
||||
<Menu.Item key="12" onClick={() => setPage("api_ref")}>
|
||||
<Menu.Item key="13" onClick={() => setPage("api_ref")}>
|
||||
<Text>API Reference</Text>
|
||||
</Menu.Item>
|
||||
<Menu.Item key="14" onClick={() => setPage("model-hub")}>
|
||||
<Menu.Item key="15" onClick={() => setPage("model-hub")}>
|
||||
<Text>Model Hub</Text>
|
||||
</Menu.Item>
|
||||
</Menu>
|
||||
|
|
|
|||
|
|
@ -1373,6 +1373,47 @@ export const adminGlobalActivity = async (
|
|||
}
|
||||
};
|
||||
|
||||
export const adminGlobalCacheActivity = async (
|
||||
accessToken: String,
|
||||
startTime: String | undefined,
|
||||
endTime: String | undefined
|
||||
) => {
|
||||
try {
|
||||
let url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/global/activity/cache_hits`
|
||||
: `/global/activity/cache_hits`;
|
||||
|
||||
if (startTime && endTime) {
|
||||
url += `?start_date=${startTime}&end_date=${endTime}`;
|
||||
}
|
||||
|
||||
const requestOptions: {
|
||||
method: string;
|
||||
headers: {
|
||||
Authorization: string;
|
||||
};
|
||||
} = {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
};
|
||||
|
||||
const response = await fetch(url, requestOptions);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
const data = await response.json();
|
||||
console.log(data);
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch spend data:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const adminGlobalActivityPerModel = async (
|
||||
accessToken: String,
|
||||
startTime: String | undefined,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue