mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
Endpoint usage POC
This commit is contained in:
yuneng-jiang
parent
e285e2b91d
commit
4c645c1ce3
7 changed files with 514 additions and 0 deletions
|
|
@ -0,0 +1,25 @@
|
|||
import React from "react";
|
||||
import EndpointUsageBarChart from "./components/EndpointUsageBarChart";
|
||||
import EndpointUsageLineChart from "./components/EndpointUsageLineChart";
|
||||
import EndpointUsageTable from "./components/EndpointUsageTable";
|
||||
import { mockEndpointData } from "./mockEndpointData";
|
||||
import { DailyData } from "../../types";
|
||||
|
||||
interface EndpointUsageProps {
|
||||
userSpendData?: {
|
||||
results: DailyData[];
|
||||
metadata: any;
|
||||
};
|
||||
}
|
||||
|
||||
const EndpointUsage: React.FC<EndpointUsageProps> = ({ userSpendData }) => {
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<EndpointUsageTable endpointData={mockEndpointData} />
|
||||
<EndpointUsageBarChart endpointData={mockEndpointData} />
|
||||
<EndpointUsageLineChart dailyData={userSpendData} endpointData={mockEndpointData} />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default EndpointUsage;
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
import React from "react";
|
||||
import { BarChart, Card, Title } from "@tremor/react";
|
||||
import { CustomLegend, CustomTooltip } from "@/components/common_components/chartUtils";
|
||||
import { MetricWithMetadata } from "../../../types";
|
||||
import { mockEndpointData } from "../mockEndpointData";
|
||||
|
||||
interface EndpointUsageBarChartProps {
|
||||
endpointData?: Record<string, MetricWithMetadata>;
|
||||
}
|
||||
|
||||
const EndpointUsageBarChart: React.FC<EndpointUsageBarChartProps> = ({ endpointData }) => {
|
||||
// Use mock data if no endpointData is provided
|
||||
const dataToUse = endpointData || mockEndpointData;
|
||||
|
||||
// Transform endpoint data into chart format
|
||||
const chartData = React.useMemo(() => {
|
||||
return Object.entries(dataToUse).map(([endpoint, data]) => ({
|
||||
endpoint,
|
||||
"metrics.successful_requests": data.metrics.successful_requests,
|
||||
"metrics.failed_requests": data.metrics.failed_requests,
|
||||
metrics: {
|
||||
successful_requests: data.metrics.successful_requests,
|
||||
failed_requests: data.metrics.failed_requests,
|
||||
},
|
||||
}));
|
||||
}, [dataToUse]);
|
||||
|
||||
const valueFormatter = (value: number) => value.toLocaleString();
|
||||
|
||||
return (
|
||||
<Card>
|
||||
<div className="flex justify-between items-center">
|
||||
<Title>Success vs Failed Requests by Endpoint</Title>
|
||||
<CustomLegend
|
||||
categories={["metrics.successful_requests", "metrics.failed_requests"]}
|
||||
colors={["green", "red"]}
|
||||
/>
|
||||
</div>
|
||||
<BarChart
|
||||
className="mt-4"
|
||||
data={chartData}
|
||||
index="endpoint"
|
||||
categories={["metrics.successful_requests", "metrics.failed_requests"]}
|
||||
colors={["green", "red"]}
|
||||
valueFormatter={valueFormatter}
|
||||
customTooltip={CustomTooltip}
|
||||
showLegend={false}
|
||||
stack={true}
|
||||
yAxisWidth={60}
|
||||
/>
|
||||
</Card>
|
||||
);
|
||||
};
|
||||
|
||||
export default EndpointUsageBarChart;
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
import React, { useMemo, useState } from "react";
|
||||
import { Card, LineChart, Title } from "@tremor/react";
|
||||
import { Segmented } from "antd";
|
||||
import { BreakdownMetrics, DailyData, MetricWithMetadata } from "../../../types";
|
||||
import { mockEndpointData } from "../mockEndpointData";
|
||||
|
||||
interface EndpointUsageLineChartProps {
|
||||
dailyData?: { results: DailyData[] };
|
||||
endpointData?: Record<string, MetricWithMetadata>;
|
||||
}
|
||||
|
||||
type TimePeriod = "7" | "30" | "90";
|
||||
|
||||
// Create mock daily data from aggregated endpoint data
|
||||
function createMockDailyData(endpointData: Record<string, MetricWithMetadata>, days: number): DailyData[] {
|
||||
const today = new Date();
|
||||
const dailyResults: DailyData[] = [];
|
||||
|
||||
for (let i = days - 1; i >= 0; i--) {
|
||||
const date = new Date(today);
|
||||
date.setDate(date.getDate() - i);
|
||||
const dateStr = date.toISOString().split("T")[0];
|
||||
|
||||
// Distribute aggregated totals across days with some variation
|
||||
const breakdown: { [key: string]: MetricWithMetadata } = {};
|
||||
Object.entries(endpointData).forEach(([endpointName, endpoint]) => {
|
||||
// Calculate average daily values
|
||||
const avgDaily = {
|
||||
spend: endpoint.metrics.spend / days,
|
||||
prompt_tokens: endpoint.metrics.prompt_tokens / days,
|
||||
completion_tokens: endpoint.metrics.completion_tokens / days,
|
||||
total_tokens: endpoint.metrics.total_tokens / days,
|
||||
api_requests: endpoint.metrics.api_requests / days,
|
||||
successful_requests: endpoint.metrics.successful_requests / days,
|
||||
failed_requests: endpoint.metrics.failed_requests / days,
|
||||
cache_read_input_tokens: endpoint.metrics.cache_read_input_tokens / days,
|
||||
cache_creation_input_tokens: endpoint.metrics.cache_creation_input_tokens / days,
|
||||
};
|
||||
|
||||
// Add simple variation (±20%) to make it look more realistic
|
||||
const variation = 0.8 + Math.random() * 0.4; // Random factor between 0.8 and 1.2
|
||||
|
||||
breakdown[endpointName] = {
|
||||
metrics: {
|
||||
spend: avgDaily.spend * variation,
|
||||
prompt_tokens: Math.round(avgDaily.prompt_tokens * variation),
|
||||
completion_tokens: Math.round(avgDaily.completion_tokens * variation),
|
||||
total_tokens: Math.round(avgDaily.total_tokens * variation),
|
||||
api_requests: Math.round(avgDaily.api_requests * variation),
|
||||
successful_requests: Math.round(avgDaily.successful_requests * variation),
|
||||
failed_requests: Math.round(avgDaily.failed_requests * variation),
|
||||
cache_read_input_tokens: Math.round(avgDaily.cache_read_input_tokens * variation),
|
||||
cache_creation_input_tokens: Math.round(avgDaily.cache_creation_input_tokens * variation),
|
||||
},
|
||||
metadata: endpoint.metadata,
|
||||
api_key_breakdown: endpoint.api_key_breakdown,
|
||||
};
|
||||
});
|
||||
|
||||
const breakdownMetrics: BreakdownMetrics = {
|
||||
models: {},
|
||||
model_groups: {},
|
||||
mcp_servers: {},
|
||||
providers: {},
|
||||
api_keys: {},
|
||||
entities: {},
|
||||
endpoints: breakdown,
|
||||
};
|
||||
|
||||
dailyResults.push({
|
||||
date: dateStr,
|
||||
metrics: {
|
||||
spend: 0,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
total_tokens: 0,
|
||||
api_requests: 0,
|
||||
successful_requests: 0,
|
||||
failed_requests: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
},
|
||||
breakdown: breakdownMetrics,
|
||||
});
|
||||
}
|
||||
|
||||
return dailyResults;
|
||||
}
|
||||
|
||||
// Transform daily data into chart format
|
||||
function transformDailyDataToChart(dailyData: DailyData[], days: number): Array<Record<string, string | number>> {
|
||||
const chartData: Array<Record<string, string | number>> = [];
|
||||
|
||||
// Get all unique endpoint names
|
||||
const endpointNames = new Set<string>();
|
||||
dailyData.forEach((day) => {
|
||||
if (day.breakdown.endpoints) {
|
||||
Object.keys(day.breakdown.endpoints).forEach((name) => endpointNames.add(name));
|
||||
}
|
||||
});
|
||||
|
||||
// Filter to the last N days based on time period
|
||||
const filteredData = dailyData.slice(-days);
|
||||
|
||||
filteredData.forEach((day) => {
|
||||
const date = new Date(day.date);
|
||||
const dateStr = date.toLocaleDateString("en-US", {
|
||||
month: "short",
|
||||
day: "numeric",
|
||||
});
|
||||
|
||||
const dataPoint: Record<string, string | number> = {
|
||||
date: dateStr,
|
||||
};
|
||||
|
||||
endpointNames.forEach((endpointName) => {
|
||||
const endpoint = day.breakdown.endpoints?.[endpointName];
|
||||
dataPoint[endpointName] = endpoint?.metrics.api_requests || 0;
|
||||
});
|
||||
|
||||
chartData.push(dataPoint);
|
||||
});
|
||||
|
||||
return chartData;
|
||||
}
|
||||
|
||||
export function EndpointUsageLineChart({ dailyData, endpointData }: EndpointUsageLineChartProps) {
|
||||
const [timePeriod, setTimePeriod] = useState<TimePeriod>("7");
|
||||
const days = parseInt(timePeriod);
|
||||
|
||||
// Use daily data if provided and has endpoints, otherwise create mock daily data from aggregated endpoint data
|
||||
const chartData = useMemo(() => {
|
||||
// Check if dailyData has endpoints breakdown
|
||||
const hasEndpointsInDailyData =
|
||||
dailyData?.results &&
|
||||
dailyData.results.some((day) => day.breakdown.endpoints && Object.keys(day.breakdown.endpoints).length > 0);
|
||||
|
||||
if (hasEndpointsInDailyData) {
|
||||
// Check if we have a reasonable number of endpoints (at least 2 for proof of concept)
|
||||
const endpointCount = new Set<string>();
|
||||
dailyData.results.forEach((day) => {
|
||||
if (day.breakdown.endpoints) {
|
||||
Object.keys(day.breakdown.endpoints).forEach((name) => endpointCount.add(name));
|
||||
}
|
||||
});
|
||||
|
||||
// If we have multiple endpoints in real data, use it; otherwise use mock data for proof of concept
|
||||
if (endpointCount.size >= 2) {
|
||||
return transformDailyDataToChart(dailyData.results, days);
|
||||
}
|
||||
}
|
||||
|
||||
// Use mock data for proof of concept
|
||||
const dataToUse = endpointData || mockEndpointData;
|
||||
const mockDaily = createMockDailyData(dataToUse, days);
|
||||
return transformDailyDataToChart(mockDaily, days);
|
||||
}, [dailyData, endpointData, days]);
|
||||
|
||||
// Get endpoint names from chart data
|
||||
const categories = useMemo(() => {
|
||||
if (chartData.length === 0) return [];
|
||||
const keys = Object.keys(chartData[0]).filter((key) => key !== "date");
|
||||
return keys;
|
||||
}, [chartData]);
|
||||
|
||||
// Tremor color palette for multiple lines
|
||||
const colors = ["blue", "cyan", "indigo", "violet", "purple", "fuchsia", "pink", "rose", "red", "orange"];
|
||||
|
||||
return (
|
||||
<Card className="mb-6">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<Title>Endpoint Usage Trends</Title>
|
||||
<Segmented
|
||||
options={[
|
||||
{
|
||||
label: "7 Days",
|
||||
value: "7",
|
||||
},
|
||||
{
|
||||
label: "30 Days",
|
||||
value: "30",
|
||||
},
|
||||
{
|
||||
label: "90 Days",
|
||||
value: "90",
|
||||
},
|
||||
]}
|
||||
value={timePeriod}
|
||||
onChange={(value) => setTimePeriod(value as TimePeriod)}
|
||||
style={{
|
||||
backgroundColor: "#f3f4f6",
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<LineChart
|
||||
className="h-80"
|
||||
data={chartData}
|
||||
index="date"
|
||||
categories={categories}
|
||||
colors={colors.slice(0, categories.length)}
|
||||
valueFormatter={(value) => value.toLocaleString()}
|
||||
showLegend={true}
|
||||
showGridLines={true}
|
||||
yAxisWidth={60}
|
||||
connectNulls={true}
|
||||
curveType="natural"
|
||||
/>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
export default EndpointUsageLineChart;
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
import React from "react";
|
||||
import { Table, Progress } from "antd";
|
||||
import type { ColumnsType } from "antd/es/table";
|
||||
import { formatNumberWithCommas } from "@/utils/dataUtils";
|
||||
import { MetricWithMetadata } from "../../../types";
|
||||
|
||||
interface EndpointUsageTableProps {
|
||||
endpointData: Record<string, MetricWithMetadata>;
|
||||
}
|
||||
|
||||
interface EndpointRow {
|
||||
key: string;
|
||||
endpoint: string;
|
||||
successful_requests: number;
|
||||
failed_requests: number;
|
||||
api_requests: number;
|
||||
total_tokens: number;
|
||||
spend: number;
|
||||
successRate: number;
|
||||
}
|
||||
|
||||
const EndpointUsageTable: React.FC<EndpointUsageTableProps> = ({ endpointData }) => {
|
||||
const calculateSuccessRate = (successful: number, total: number): number => {
|
||||
if (total === 0) return 0;
|
||||
return (successful / total) * 100;
|
||||
};
|
||||
|
||||
const dataSource: EndpointRow[] = Object.entries(endpointData).map(([endpoint, data]) => ({
|
||||
key: endpoint,
|
||||
endpoint,
|
||||
successful_requests: data.metrics.successful_requests,
|
||||
failed_requests: data.metrics.failed_requests,
|
||||
api_requests: data.metrics.api_requests,
|
||||
total_tokens: data.metrics.total_tokens,
|
||||
spend: data.metrics.spend,
|
||||
successRate: calculateSuccessRate(data.metrics.successful_requests, data.metrics.api_requests),
|
||||
}));
|
||||
|
||||
const columns: ColumnsType<EndpointRow> = [
|
||||
{
|
||||
title: "Endpoint",
|
||||
dataIndex: "endpoint",
|
||||
key: "endpoint",
|
||||
render: (text: string) => <span className="font-medium">{text}</span>,
|
||||
},
|
||||
{
|
||||
title: "Successful / Failed",
|
||||
key: "requests",
|
||||
render: (_: any, record: EndpointRow) => {
|
||||
const successPercentage =
|
||||
record.api_requests > 0 ? (record.successful_requests / record.api_requests) * 100 : 0;
|
||||
const failurePercentage = record.api_requests > 0 ? (record.failed_requests / record.api_requests) * 100 : 0;
|
||||
const totalPercentage = successPercentage + failurePercentage;
|
||||
|
||||
const strokeColorConfig: Record<string, string> = {
|
||||
"0%": "#22c55e",
|
||||
};
|
||||
if (successPercentage > 0 && successPercentage < 100) {
|
||||
strokeColorConfig[`${successPercentage}%`] = "#22c55e";
|
||||
strokeColorConfig[`${successPercentage + 0.01}%`] = "#ef4444";
|
||||
}
|
||||
strokeColorConfig["100%"] = failurePercentage > 0 ? "#ef4444" : "#22c55e";
|
||||
|
||||
return (
|
||||
<div className="flex items-center space-x-3">
|
||||
<div className="flex-1 relative">
|
||||
<Progress percent={totalPercentage} size="small" strokeColor={strokeColorConfig} showInfo={false} />
|
||||
</div>
|
||||
<div className="flex items-center space-x-2 text-sm min-w-[100px]">
|
||||
<span className="text-green-600 font-medium">{record.successful_requests.toLocaleString()}</span>
|
||||
<span className="text-gray-400">/</span>
|
||||
<span className="text-red-600 font-medium">{record.failed_requests.toLocaleString()}</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Total Request",
|
||||
dataIndex: "api_requests",
|
||||
key: "api_requests",
|
||||
render: (value: number) => value.toLocaleString(),
|
||||
},
|
||||
{
|
||||
title: "Success Rate",
|
||||
dataIndex: "successRate",
|
||||
key: "successRate",
|
||||
render: (value: number) => {
|
||||
const successRateStr = value.toFixed(2);
|
||||
return (
|
||||
<span
|
||||
className={
|
||||
value >= 95
|
||||
? "text-green-600 font-medium"
|
||||
: value >= 80
|
||||
? "text-yellow-600 font-medium"
|
||||
: "text-red-600 font-medium"
|
||||
}
|
||||
>
|
||||
{successRateStr}%
|
||||
</span>
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
title: "Total Tokens",
|
||||
dataIndex: "total_tokens",
|
||||
key: "total_tokens",
|
||||
render: (value: number) => value.toLocaleString(),
|
||||
},
|
||||
{
|
||||
title: "Spend",
|
||||
dataIndex: "spend",
|
||||
key: "spend",
|
||||
render: (value: number) => `$${formatNumberWithCommas(value, 2)}`,
|
||||
},
|
||||
];
|
||||
|
||||
return <Table columns={columns} dataSource={dataSource} pagination={false} />;
|
||||
};
|
||||
|
||||
export default EndpointUsageTable;
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
import { MetricWithMetadata } from "../../types";
|
||||
|
||||
export const mockEndpointData: Record<string, MetricWithMetadata> = {
|
||||
"/chat/completions": {
|
||||
metrics: {
|
||||
spend: 1250.45,
|
||||
prompt_tokens: 150000,
|
||||
completion_tokens: 45000,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
total_tokens: 195000,
|
||||
successful_requests: 1250,
|
||||
failed_requests: 45,
|
||||
api_requests: 1295,
|
||||
},
|
||||
metadata: {},
|
||||
api_key_breakdown: {},
|
||||
},
|
||||
"/audio/speech": {
|
||||
metrics: {
|
||||
spend: 320.78,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
total_tokens: 0,
|
||||
successful_requests: 890,
|
||||
failed_requests: 12,
|
||||
api_requests: 902,
|
||||
},
|
||||
metadata: {},
|
||||
api_key_breakdown: {},
|
||||
},
|
||||
"/audio/transcriptions": {
|
||||
metrics: {
|
||||
spend: 85.23,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
total_tokens: 0,
|
||||
successful_requests: 234,
|
||||
failed_requests: 8,
|
||||
api_requests: 242,
|
||||
},
|
||||
metadata: {},
|
||||
api_key_breakdown: {},
|
||||
},
|
||||
"/embeddings": {
|
||||
metrics: {
|
||||
spend: 450.12,
|
||||
prompt_tokens: 200000,
|
||||
completion_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
total_tokens: 200000,
|
||||
successful_requests: 2100,
|
||||
failed_requests: 23,
|
||||
api_requests: 2123,
|
||||
},
|
||||
metadata: {},
|
||||
api_key_breakdown: {},
|
||||
},
|
||||
"/images/generations": {
|
||||
metrics: {
|
||||
spend: 180.56,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
total_tokens: 0,
|
||||
successful_requests: 156,
|
||||
failed_requests: 5,
|
||||
api_requests: 161,
|
||||
},
|
||||
metadata: {},
|
||||
api_key_breakdown: {},
|
||||
},
|
||||
"/moderations": {
|
||||
metrics: {
|
||||
spend: 95.34,
|
||||
prompt_tokens: 50000,
|
||||
completion_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
total_tokens: 50000,
|
||||
successful_requests: 567,
|
||||
failed_requests: 3,
|
||||
api_requests: 570,
|
||||
},
|
||||
metadata: {},
|
||||
api_key_breakdown: {},
|
||||
},
|
||||
};
|
||||
|
|
@ -53,6 +53,7 @@ import EntityUsage, { EntityList } from "./EntityUsage/EntityUsage";
|
|||
import TopKeyView from "./EntityUsage/TopKeyView";
|
||||
import { UsageOption, UsageViewSelect } from "./UsageViewSelect/UsageViewSelect";
|
||||
import { useCurrentUser } from "@/app/(dashboard)/hooks/users/useCurrentUser";
|
||||
import EndpointUsage from "./EndpointUsage/EndpointUsage";
|
||||
|
||||
interface UsagePageProps {
|
||||
teams: Team[];
|
||||
|
|
@ -439,6 +440,7 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
|
|||
<Tab>Model Activity</Tab>
|
||||
<Tab>Key Activity</Tab>
|
||||
<Tab>MCP Server Activity</Tab>
|
||||
<Tab>Endpoint Activity</Tab>
|
||||
</TabList>
|
||||
<Button
|
||||
onClick={() => setIsGlobalExportModalOpen(true)}
|
||||
|
|
@ -728,6 +730,9 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
|
|||
<TabPanel>
|
||||
<ActivityMetrics modelMetrics={mcpServerMetrics} />
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
<EndpointUsage userSpendData={userSpendData} />
|
||||
</TabPanel>
|
||||
</TabPanels>
|
||||
</TabGroup>
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ export interface BreakdownMetrics {
|
|||
providers: { [key: string]: MetricWithMetadata };
|
||||
api_keys: { [key: string]: KeyMetricWithMetadata };
|
||||
entities: { [key: string]: MetricWithMetadata };
|
||||
endpoints?: { [key: string]: MetricWithMetadata };
|
||||
}
|
||||
|
||||
export interface MetricWithMetadata {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue