mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
refactor(ui): consolidate model leaderboard into Usage (#51426)
This commit is contained in:
moe-berri
GitHub
parent
8ba416ab2e
commit
4ee55a28f7
11 changed files with 12 additions and 746 deletions
|
|
@ -39,7 +39,7 @@ const LEGACY_PAGE_ROUTES: ReadonlyMap<string, string> = new Map(
|
|||
new_usage: "usage",
|
||||
usage: "usage",
|
||||
"cost-optimization": "cost-optimization",
|
||||
"model-insights": "model-insights",
|
||||
"model-insights": "usage",
|
||||
agents: "agents",
|
||||
"router-settings": "router-settings",
|
||||
users: "users",
|
||||
|
|
|
|||
|
|
@ -1,125 +0,0 @@
|
|||
import { render, screen, waitFor } from "@testing-library/react";
|
||||
import userEvent from "@testing-library/user-event";
|
||||
import type React from "react";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import ModelInsightsView from "./ModelInsightsView";
|
||||
import { apiClient } from "@/components/networking";
|
||||
|
||||
vi.mock("@/components/networking", () => ({ apiClient: { get: vi.fn() } }));
|
||||
vi.mock("@/components/ui/chart", () => ({
|
||||
ChartContainer: ({ children }: { children: React.ReactNode }) => <div>{children}</div>,
|
||||
ChartTooltip: () => null,
|
||||
ChartTooltipContent: () => null,
|
||||
}));
|
||||
vi.mock("recharts", () => ({
|
||||
Bar: () => null,
|
||||
BarChart: ({ children, data }: { children: React.ReactNode; data: { date: string }[] }) => (
|
||||
<div data-testid="usage-chart" data-first={data[0]?.date} data-buckets={data.length}>
|
||||
{children}
|
||||
</div>
|
||||
),
|
||||
CartesianGrid: () => null,
|
||||
XAxis: () => null,
|
||||
YAxis: () => null,
|
||||
}));
|
||||
|
||||
const metrics = {
|
||||
model_group: "fast-chat",
|
||||
model: "openai/gpt-5.4-mini",
|
||||
provider: "openai",
|
||||
spend: 2.5,
|
||||
prompt_tokens: 1000,
|
||||
completion_tokens: 2000,
|
||||
requests: 12,
|
||||
successful_requests: 12,
|
||||
failed_requests: 0,
|
||||
};
|
||||
|
||||
const response = {
|
||||
start_date: "2025-09-29",
|
||||
end_date: "2026-09-28",
|
||||
top_models: [metrics],
|
||||
daily: [{ ...metrics, date: "2026-09-28" }],
|
||||
daily_totals: [{ date: "2026-09-28", spend: 2.5, prompt_tokens: 1000, completion_tokens: 2000, requests: 12 }],
|
||||
};
|
||||
|
||||
describe("ModelInsightsView", () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(apiClient.get).mockReset();
|
||||
vi.mocked(apiClient.get).mockResolvedValue(response);
|
||||
});
|
||||
|
||||
it("shows the ranking with share from the API response", async () => {
|
||||
render(<ModelInsightsView accessToken="token" />);
|
||||
|
||||
expect(await screen.findByText("fast-chat")).toBeInTheDocument();
|
||||
expect(screen.getByText("by openai")).toBeInTheDocument();
|
||||
expect(screen.getAllByText("100.0%")).toHaveLength(1);
|
||||
expect(screen.queryByText("Top models by task")).not.toBeInTheDocument();
|
||||
expect(screen.getByRole("tab", { name: "tokens" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByRole("tab", { name: "log" })).toBeInTheDocument();
|
||||
expect(apiClient.get).toHaveBeenCalledWith("/model-insights", {
|
||||
accessToken: "token",
|
||||
query: { metric: "tokens" },
|
||||
});
|
||||
expect(apiClient.get).not.toHaveBeenCalledWith("/model-insights/tasks", expect.anything());
|
||||
});
|
||||
|
||||
it("refetches with the selected metric so top models are ranked by it", async () => {
|
||||
render(<ModelInsightsView accessToken="token" />);
|
||||
await screen.findByText("fast-chat");
|
||||
|
||||
await userEvent.click(screen.getByRole("tab", { name: "requests" }));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(apiClient.get).toHaveBeenCalledWith("/model-insights", {
|
||||
accessToken: "token",
|
||||
query: { metric: "requests" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("shows the API error instead of loading forever", async () => {
|
||||
vi.mocked(apiClient.get).mockRejectedValue(new Error("Only proxy admins can view deployment-wide model insights"));
|
||||
render(<ModelInsightsView accessToken="token" />);
|
||||
|
||||
expect(await screen.findByText("Could not load model insights")).toBeInTheDocument();
|
||||
expect(screen.getByText("Only proxy admins can view deployment-wide model insights")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("keeps the previous ranking, dimmed, until the new metric's data arrives", async () => {
|
||||
render(<ModelInsightsView accessToken="token" />);
|
||||
await screen.findByText("fast-chat");
|
||||
let resolve: (value: typeof response) => void = () => {};
|
||||
vi.mocked(apiClient.get).mockImplementation(() => new Promise((done) => (resolve = done as typeof resolve)));
|
||||
|
||||
await userEvent.click(screen.getByRole("tab", { name: "spend" }));
|
||||
|
||||
expect(
|
||||
screen.getByText("Share of tokens, with the change between the first and second half of the period"),
|
||||
).toBeInTheDocument();
|
||||
|
||||
resolve(response);
|
||||
expect(
|
||||
await screen.findByText("Share of spend, with the change between the first and second half of the period"),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("charts one bar per day by default and switches to weekly bars", async () => {
|
||||
render(<ModelInsightsView accessToken="token" />);
|
||||
await screen.findByText("fast-chat");
|
||||
const chart = screen.getByTestId("usage-chart");
|
||||
|
||||
expect(screen.getByRole("tab", { name: "Daily" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(chart).toHaveAttribute("data-buckets", "30");
|
||||
expect(chart).toHaveAttribute("data-first", "2026-08-30");
|
||||
expect(screen.getByText("Daily tokens across your gateway")).toBeInTheDocument();
|
||||
|
||||
await userEvent.click(screen.getByRole("tab", { name: "Weekly" }));
|
||||
|
||||
expect(chart).toHaveAttribute("data-buckets", "12");
|
||||
expect(chart).toHaveAttribute("data-first", "2026-07-13");
|
||||
expect(screen.getByText("Weekly tokens across your gateway")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,247 +0,0 @@
|
|||
"use client";
|
||||
|
||||
import { Page } from "@/components/shared/Page";
|
||||
import type { ParseKeys } from "i18next";
|
||||
import React from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { ArrowDownRight, ArrowUpRight, BarChart3, Minus } from "lucide-react";
|
||||
import { StackedUsageChart } from "@/components/shared/charts";
|
||||
|
||||
import { apiClient } from "@/components/networking";
|
||||
import { extractErrorMessage } from "@/utils/errorUtils";
|
||||
import { ProviderLogo } from "@/components/molecules/models/ProviderLogo";
|
||||
import { PageHeader, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader";
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert";
|
||||
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import {
|
||||
buildBucketTotals,
|
||||
buildSeries,
|
||||
formatMetric,
|
||||
Granularity,
|
||||
Metric,
|
||||
ModelInsightsResponse,
|
||||
modelOrder,
|
||||
rankModels,
|
||||
RankedModel,
|
||||
} from "./modelInsightsData";
|
||||
|
||||
const SCALES = ["linear", "log"] as const;
|
||||
const GRANULARITIES = ["day", "week"] as const;
|
||||
const METRICS = ["requests", "spend", "tokens"] as const;
|
||||
const GRANULARITY_LABELS: Record<Granularity, ParseKeys> = {
|
||||
day: "modelInsights.modelInsightsView.granularity.day",
|
||||
week: "modelInsights.modelInsightsView.granularity.week",
|
||||
};
|
||||
const METRIC_LABELS: Record<Metric, ParseKeys> = {
|
||||
requests: "modelInsights.modelInsightsView.metrics.requests",
|
||||
spend: "modelInsights.modelInsightsView.metrics.spend",
|
||||
tokens: "modelInsights.modelInsightsView.metrics.tokens",
|
||||
};
|
||||
const RANKING_ROWS = 5;
|
||||
|
||||
type Scale = (typeof SCALES)[number];
|
||||
|
||||
const formatDelta = (value: number) => `${value > 0 ? "+" : ""}${value.toFixed(1)}`;
|
||||
|
||||
const DeltaBadge = ({ value }: { value: number }) => {
|
||||
if (Math.abs(value) < 0.05) {
|
||||
return (
|
||||
<span className="flex items-center justify-end gap-1 text-xs text-muted-foreground">
|
||||
<Minus className="size-3" /> 0.0
|
||||
</span>
|
||||
);
|
||||
}
|
||||
const up = value > 0;
|
||||
const Icon = up ? ArrowUpRight : ArrowDownRight;
|
||||
return (
|
||||
<span className={`flex items-center justify-end gap-1 text-xs ${up ? "text-emerald-600" : "text-red-600"}`}>
|
||||
<Icon className="size-3" /> {formatDelta(value)}
|
||||
</span>
|
||||
);
|
||||
};
|
||||
|
||||
const RankingRow = ({ model, rank }: { model: RankedModel; rank: number }) => {
|
||||
const { t } = useTranslation();
|
||||
return (
|
||||
<li className="grid grid-cols-[1.5rem_2.5rem_1fr_auto] items-center gap-3 py-2">
|
||||
<span className="text-sm tabular-nums text-muted-foreground">{rank}</span>
|
||||
<ProviderLogo provider={model.provider} className="size-9 rounded-md border p-1" />
|
||||
<div className="min-w-0">
|
||||
<p className="truncate font-medium">{model.model_group}</p>
|
||||
<p className="truncate text-sm text-muted-foreground">
|
||||
{t("modelInsights.modelInsightsView.byProvider", { provider: model.provider })}
|
||||
</p>
|
||||
</div>
|
||||
<div className="text-right">
|
||||
<p className="font-medium tabular-nums">{model.share.toFixed(1)}%</p>
|
||||
<DeltaBadge value={model.delta} />
|
||||
</div>
|
||||
</li>
|
||||
);
|
||||
};
|
||||
|
||||
export default function ModelInsightsView({ accessToken }: { accessToken: string | null }) {
|
||||
const { t } = useTranslation();
|
||||
const [loaded, setLoaded] = React.useState<{ metric: Metric; response: ModelInsightsResponse } | null>(null);
|
||||
const [metric, setMetric] = React.useState<Metric>("tokens");
|
||||
const [scale, setScale] = React.useState<Scale>("linear");
|
||||
const [granularity, setGranularity] = React.useState<Granularity>("day");
|
||||
const [error, setError] = React.useState<string | null>(null);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!accessToken) return;
|
||||
let cancelled = false;
|
||||
apiClient
|
||||
.get<ModelInsightsResponse>("/model-insights", { accessToken, query: { metric } })
|
||||
.then((response) => {
|
||||
if (cancelled) return;
|
||||
setError(null);
|
||||
setLoaded({ metric, response });
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (!cancelled) setError(extractErrorMessage(err));
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [accessToken, metric]);
|
||||
|
||||
const data = loaded?.response ?? null;
|
||||
const shown = loaded?.metric ?? metric;
|
||||
const isStale = loaded !== null && loaded.metric !== metric;
|
||||
const range = React.useMemo(() => ({ start: data?.start_date ?? "", end: data?.end_date ?? "" }), [data]);
|
||||
const models = React.useMemo(() => (data ? modelOrder(data.daily, shown) : []), [data, shown]);
|
||||
const series = React.useMemo(
|
||||
() => (data ? buildSeries(data.daily, models, shown, { ...range, granularity }) : []),
|
||||
[data, models, shown, range, granularity],
|
||||
);
|
||||
const bucketTotals = React.useMemo(
|
||||
() => (data ? buildBucketTotals(data.daily_totals, shown, { ...range, granularity }) : new Map<string, number>()),
|
||||
[data, shown, range, granularity],
|
||||
);
|
||||
const ranking = React.useMemo(
|
||||
() => (data ? rankModels(data.top_models, data.daily, shown, range) : []),
|
||||
[data, shown, range],
|
||||
);
|
||||
|
||||
if (error) {
|
||||
return (
|
||||
<div className="p-8">
|
||||
<Alert variant="destructive">
|
||||
<AlertTitle>{t("modelInsights.modelInsightsView.loadError")}</AlertTitle>
|
||||
<AlertDescription>{error}</AlertDescription>
|
||||
</Alert>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!data) {
|
||||
return (
|
||||
<div className="space-y-6 p-8">
|
||||
<Skeleton className="h-16 w-96" />
|
||||
<Skeleton className="h-96 w-full" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Page>
|
||||
<PageHeader>
|
||||
<PageHeaderTitle>
|
||||
<BarChart3 />
|
||||
{t("modelInsights.modelInsightsView.title")}
|
||||
</PageHeaderTitle>
|
||||
<PageHeaderDescription>
|
||||
{t("modelInsights.modelInsightsView.description", { start: data.start_date, end: data.end_date })}
|
||||
</PageHeaderDescription>
|
||||
</PageHeader>
|
||||
|
||||
<Card aria-busy={isStale} className={isStale ? "opacity-60 transition-opacity" : "transition-opacity"}>
|
||||
<CardHeader className="flex-row items-start justify-between space-y-0">
|
||||
<div>
|
||||
<CardTitle>{t("modelInsights.modelInsightsView.topModels.title")}</CardTitle>
|
||||
<CardDescription>
|
||||
{t("modelInsights.modelInsightsView.topModels.description", {
|
||||
granularity: t(GRANULARITY_LABELS[granularity]),
|
||||
metric: t(METRIC_LABELS[shown]),
|
||||
})}
|
||||
</CardDescription>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<Tabs value={metric} onValueChange={(value) => setMetric(value as Metric)}>
|
||||
<TabsList>
|
||||
{METRICS.map((value) => (
|
||||
<TabsTrigger key={value} value={value} className="capitalize">
|
||||
{t(METRIC_LABELS[value])}
|
||||
</TabsTrigger>
|
||||
))}
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
<Tabs value={granularity} onValueChange={(value) => setGranularity(value as Granularity)}>
|
||||
<TabsList aria-label={t("modelInsights.modelInsightsView.bucketSize")}>
|
||||
{GRANULARITIES.map((value) => (
|
||||
<TabsTrigger key={value} value={value}>
|
||||
{t(GRANULARITY_LABELS[value])}
|
||||
</TabsTrigger>
|
||||
))}
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
<Tabs value={scale} onValueChange={(value) => setScale(value as Scale)}>
|
||||
<TabsList>
|
||||
{SCALES.map((value) => (
|
||||
<TabsTrigger key={value} value={value} className="capitalize">
|
||||
{value}
|
||||
</TabsTrigger>
|
||||
))}
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<StackedUsageChart
|
||||
data={series}
|
||||
series={models}
|
||||
xKey="date"
|
||||
scale={scale}
|
||||
format={(value) => formatMetric(value, shown)}
|
||||
totalLabel={t("modelInsights.modelInsightsView.gatewayTotal")}
|
||||
totalFor={(label) => bucketTotals.get(label) ?? 0}
|
||||
/>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card aria-busy={isStale} className={isStale ? "opacity-60 transition-opacity" : "transition-opacity"}>
|
||||
<CardHeader>
|
||||
<CardTitle>{t("modelInsights.modelInsightsView.leaderboard.title")}</CardTitle>
|
||||
<CardDescription>
|
||||
{t("modelInsights.modelInsightsView.leaderboard.description", { metric: t(METRIC_LABELS[shown]) })}
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="grid gap-x-12 md:grid-cols-2">
|
||||
<ol className="divide-y">
|
||||
{ranking.slice(0, RANKING_ROWS).map((model, index) => (
|
||||
<RankingRow key={model.model_group} model={model} rank={index + 1} />
|
||||
))}
|
||||
</ol>
|
||||
<ol className="divide-y">
|
||||
{ranking.slice(RANKING_ROWS).map((model, index) => (
|
||||
<RankingRow key={model.model_group} model={model} rank={RANKING_ROWS + index + 1} />
|
||||
))}
|
||||
</ol>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>{t("modelInsights.modelInsightsView.costPerSession.title")}</CardTitle>
|
||||
<CardDescription>{t("modelInsights.modelInsightsView.costPerSession.description")}</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<p className="text-sm text-muted-foreground">{t("modelInsights.modelInsightsView.costPerSession.hint")}</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</Page>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,169 +0,0 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { buildBucketTotals, buildSeries, DailyMetric, formatMetric, modelOrder, rankModels } from "./modelInsightsData";
|
||||
|
||||
const row = (over: Partial<DailyMetric>): DailyMetric => ({
|
||||
model_group: "a",
|
||||
model: "a",
|
||||
provider: "openai",
|
||||
date: "2026-01-01",
|
||||
spend: 0,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
requests: 0,
|
||||
successful_requests: 0,
|
||||
failed_requests: 0,
|
||||
...over,
|
||||
});
|
||||
|
||||
describe("buildSeries", () => {
|
||||
const range = { start: "2026-01-01", end: "2026-01-15" };
|
||||
|
||||
it("sums days into 7-day buckets per model", () => {
|
||||
const rows = [
|
||||
row({ date: "2026-01-01", requests: 1 }),
|
||||
row({ date: "2026-01-07", requests: 2 }),
|
||||
row({ date: "2026-01-08", requests: 4 }),
|
||||
row({ date: "2026-01-02", model_group: "b", requests: 8 }),
|
||||
];
|
||||
expect(buildSeries(rows, ["a", "b"], "requests", { ...range, granularity: "week" })).toEqual([
|
||||
{ date: "2026-01-01", a: 3, b: 8 },
|
||||
{ date: "2026-01-08", a: 4, b: 0 },
|
||||
{ date: "2026-01-15", a: 0, b: 0 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps weeks with no usage as zero instead of dropping them", () => {
|
||||
const rows = [row({ date: "2026-01-01", requests: 1 }), row({ date: "2026-01-15", requests: 2 })];
|
||||
expect(
|
||||
buildSeries(rows, ["a"], "requests", { ...range, granularity: "week" }).map((week) => [week.date, week.a]),
|
||||
).toEqual([
|
||||
["2026-01-01", 1],
|
||||
["2026-01-08", 0],
|
||||
["2026-01-15", 2],
|
||||
]);
|
||||
});
|
||||
|
||||
it("gives every day its own bucket with that day's token total", () => {
|
||||
const rows = [
|
||||
row({ date: "2026-01-01", prompt_tokens: 100, completion_tokens: 50 }),
|
||||
row({ date: "2026-01-01", prompt_tokens: 10, completion_tokens: 5 }),
|
||||
row({ date: "2026-01-03", prompt_tokens: 7, completion_tokens: 3 }),
|
||||
];
|
||||
const daily = buildSeries(rows, ["a"], "tokens", { start: "2026-01-01", end: "2026-01-03", granularity: "day" });
|
||||
expect(daily).toEqual([
|
||||
{ date: "2026-01-01", a: 165 },
|
||||
{ date: "2026-01-02", a: 0 },
|
||||
{ date: "2026-01-03", a: 10 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("starts the chart at the first bucket with usage so bars stay wide on a long range", () => {
|
||||
const rows = [row({ date: "2026-03-10", requests: 1 }), row({ date: "2026-03-20", requests: 2 })];
|
||||
const daily = buildSeries(rows, ["a"], "requests", { start: "2025-03-21", end: "2026-03-20", granularity: "day" });
|
||||
expect(daily[0]).toEqual({ date: "2026-02-19", a: 0 });
|
||||
expect(daily).toHaveLength(30);
|
||||
expect(daily.at(-1)).toEqual({ date: "2026-03-20", a: 2 });
|
||||
|
||||
const early = [row({ date: "2025-12-01", requests: 1 }), ...rows];
|
||||
const fromFirstUse = buildSeries(early, ["a"], "requests", {
|
||||
start: "2025-03-21",
|
||||
end: "2026-03-20",
|
||||
granularity: "day",
|
||||
});
|
||||
expect(fromFirstUse[0]).toEqual({ date: "2025-12-01", a: 1 });
|
||||
expect(fromFirstUse.at(-1)?.date).toBe("2026-03-20");
|
||||
});
|
||||
|
||||
it("keeps weekly buckets on the original grid when trimming idle weeks", () => {
|
||||
const rows = [row({ date: "2026-03-20", requests: 3 })];
|
||||
const weekly = buildSeries(rows, ["a"], "requests", {
|
||||
start: "2025-03-21",
|
||||
end: "2026-03-20",
|
||||
granularity: "week",
|
||||
});
|
||||
expect(weekly).toHaveLength(12);
|
||||
expect(weekly.map((week) => (Date.parse(String(week.date)) - Date.parse("2025-03-21")) % (7 * 86_400_000))).toEqual(
|
||||
Array(12).fill(0),
|
||||
);
|
||||
expect(weekly.at(-1)).toEqual({ date: "2026-03-20", a: 3 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildBucketTotals", () => {
|
||||
const total = (date: string, prompt_tokens: number) => ({
|
||||
date,
|
||||
spend: 0,
|
||||
prompt_tokens,
|
||||
completion_tokens: 1,
|
||||
requests: 0,
|
||||
});
|
||||
const totals = [total("2026-01-01", 9), total("2026-01-03", 4), total("2026-01-08", 99)];
|
||||
|
||||
it("keys each day's gateway-wide total by its own date", () => {
|
||||
const daily = buildBucketTotals(totals, "tokens", { start: "2026-01-01", end: "2026-01-08", granularity: "day" });
|
||||
expect([...daily]).toEqual([
|
||||
["2026-01-01", 10],
|
||||
["2026-01-03", 5],
|
||||
["2026-01-08", 100],
|
||||
]);
|
||||
});
|
||||
|
||||
it("sums days into the same week start used by the chart's x-axis", () => {
|
||||
const window = { start: "2026-01-01", end: "2026-01-08", granularity: "week" } as const;
|
||||
const weekly = buildBucketTotals(totals, "tokens", window);
|
||||
expect([...weekly]).toEqual([
|
||||
["2026-01-01", 15],
|
||||
["2026-01-08", 100],
|
||||
]);
|
||||
expect([...weekly.keys()]).toEqual(buildSeries([], [], "tokens", window).map((bucket) => bucket.date));
|
||||
});
|
||||
});
|
||||
|
||||
describe("modelOrder", () => {
|
||||
it("orders models by the selected metric, largest first", () => {
|
||||
const rows = [row({ model_group: "a", spend: 1, requests: 9 }), row({ model_group: "b", spend: 5, requests: 1 })];
|
||||
expect(modelOrder(rows, "spend")).toEqual(["b", "a"]);
|
||||
expect(modelOrder(rows, "requests")).toEqual(["a", "b"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("rankModels", () => {
|
||||
const range = { start: "2026-01-01", end: "2026-01-10" };
|
||||
const totals = [row({ model_group: "a", requests: 40 }), row({ model_group: "b", requests: 40 })];
|
||||
|
||||
it("computes share and the change in share between the first and second half of the range", () => {
|
||||
const daily = [
|
||||
row({ date: "2026-01-01", model_group: "a", requests: 30 }),
|
||||
row({ date: "2026-01-01", model_group: "b", requests: 10 }),
|
||||
row({ date: "2026-01-10", model_group: "a", requests: 10 }),
|
||||
row({ date: "2026-01-10", model_group: "b", requests: 30 }),
|
||||
];
|
||||
const ranked = rankModels(totals, daily, "requests", range);
|
||||
expect(ranked.find((m) => m.model_group === "a")).toMatchObject({ share: 50, delta: -50 });
|
||||
expect(ranked.find((m) => m.model_group === "b")).toMatchObject({ share: 50, delta: 50 });
|
||||
});
|
||||
|
||||
it("splits at the middle of the range, not the middle of the days that had usage", () => {
|
||||
const daily = [
|
||||
row({ date: "2026-01-01", model_group: "a", requests: 10 }),
|
||||
row({ date: "2026-01-02", model_group: "b", requests: 10 }),
|
||||
row({ date: "2026-01-03", model_group: "b", requests: 10 }),
|
||||
];
|
||||
const ranked = rankModels(totals, daily, "requests", range);
|
||||
expect(ranked.find((m) => m.model_group === "a")?.delta).toBe(0);
|
||||
});
|
||||
|
||||
it("shows no change when one half of the range has no usage to compare against", () => {
|
||||
const daily = [row({ date: "2026-01-10", model_group: "a", requests: 10 })];
|
||||
const ranked = rankModels(totals, daily, "requests", range);
|
||||
expect(ranked.map((m) => m.delta)).toEqual([0, 0]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatMetric", () => {
|
||||
it("formats spend as currency and counts compactly", () => {
|
||||
expect(formatMetric(12.5, "spend")).toBe("$12.50");
|
||||
expect(formatMetric(1_500_000, "tokens")).toBe("1.5M");
|
||||
});
|
||||
});
|
||||
|
|
@ -1,146 +0,0 @@
|
|||
export type Metric = "requests" | "spend" | "tokens";
|
||||
|
||||
export type ModelMetric = {
|
||||
model_group: string;
|
||||
model: string;
|
||||
provider: string;
|
||||
spend: number;
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
requests: number;
|
||||
successful_requests: number;
|
||||
failed_requests: number;
|
||||
};
|
||||
export type DailyMetric = ModelMetric & { date: string };
|
||||
type Usage = Pick<ModelMetric, "spend" | "prompt_tokens" | "completion_tokens" | "requests">;
|
||||
export type DailyTotal = Usage & { date: string };
|
||||
export type ModelInsightsResponse = {
|
||||
start_date: string;
|
||||
end_date: string;
|
||||
daily: DailyMetric[];
|
||||
daily_totals: DailyTotal[];
|
||||
top_models: ModelMetric[];
|
||||
};
|
||||
|
||||
export type RankedModel = { model_group: string; provider: string; share: number; delta: number };
|
||||
export type Granularity = "day" | "week";
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
const BUCKET_DAYS: Record<Granularity, number> = { day: 1, week: 7 };
|
||||
const MIN_VISIBLE_BUCKETS: Record<Granularity, number> = { day: 30, week: 12 };
|
||||
|
||||
export const metricValue = (row: Usage, metric: Metric) => {
|
||||
if (metric === "requests") return row.requests;
|
||||
if (metric === "spend") return row.spend;
|
||||
return row.prompt_tokens + row.completion_tokens;
|
||||
};
|
||||
|
||||
const COMPACT_SPEND_FROM = 10_000;
|
||||
|
||||
export const formatMetric = (value: number, metric: Metric) => {
|
||||
if (metric === "spend") {
|
||||
const compact = value >= COMPACT_SPEND_FROM;
|
||||
const options: Intl.NumberFormatOptions = {
|
||||
style: "currency",
|
||||
currency: "USD",
|
||||
notation: compact ? "compact" : "standard",
|
||||
maximumFractionDigits: compact ? 1 : 2,
|
||||
};
|
||||
return new Intl.NumberFormat("en-US", options).format(value);
|
||||
}
|
||||
return new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }).format(value);
|
||||
};
|
||||
|
||||
const toDay = (date: string) => Date.parse(`${date}T00:00:00Z`);
|
||||
const isoDay = (ms: number) => new Date(ms).toISOString().slice(0, 10);
|
||||
|
||||
export type DateRange = { start: string; end: string };
|
||||
|
||||
export const modelOrder = (rows: DailyMetric[], metric: Metric) => {
|
||||
const totals = new Map<string, number>();
|
||||
for (const row of rows) totals.set(row.model_group, (totals.get(row.model_group) ?? 0) + metricValue(row, metric));
|
||||
return [...totals.entries()].sort((a, b) => b[1] - a[1]).map(([model]) => model);
|
||||
};
|
||||
|
||||
export type SeriesWindow = DateRange & { granularity: Granularity };
|
||||
|
||||
export const buildSeries = (rows: DailyMetric[], models: string[], metric: Metric, window: SeriesWindow) => {
|
||||
const bucketMs = BUCKET_DAYS[window.granularity] * DAY_MS;
|
||||
const origin = toDay(window.start);
|
||||
const bucketCount = Math.floor((toDay(window.end) - origin) / bucketMs) + 1;
|
||||
const buckets = Array.from({ length: bucketCount }, (_, index) => ({
|
||||
date: isoDay(origin + index * bucketMs),
|
||||
...Object.fromEntries(models.map((model) => [model, 0])),
|
||||
})) as Record<string, number | string>[];
|
||||
for (const row of rows) {
|
||||
const bucket = buckets[Math.floor((toDay(row.date) - origin) / bucketMs)];
|
||||
if (bucket) bucket[row.model_group] = Number(bucket[row.model_group] ?? 0) + metricValue(row, metric);
|
||||
}
|
||||
const firstActive = buckets.findIndex((bucket) => models.some((model) => Number(bucket[model]) > 0));
|
||||
const visibleFrom = Math.min(
|
||||
firstActive === -1 ? buckets.length : firstActive,
|
||||
buckets.length - MIN_VISIBLE_BUCKETS[window.granularity],
|
||||
);
|
||||
return buckets.slice(Math.max(0, visibleFrom));
|
||||
};
|
||||
|
||||
export const buildBucketTotals = (totals: DailyTotal[], metric: Metric, window: SeriesWindow) => {
|
||||
const bucketMs = BUCKET_DAYS[window.granularity] * DAY_MS;
|
||||
const origin = toDay(window.start);
|
||||
const byBucket = new Map<string, number>();
|
||||
for (const row of totals) {
|
||||
const bucket = isoDay(origin + Math.floor((toDay(row.date) - origin) / bucketMs) * bucketMs);
|
||||
byBucket.set(bucket, (byBucket.get(bucket) ?? 0) + metricValue(row, metric));
|
||||
}
|
||||
return byBucket;
|
||||
};
|
||||
|
||||
const shareByModel = (rows: { model_group: string; provider: string }[], values: number[]) => {
|
||||
const totals = new Map<string, { provider: string; value: number }>();
|
||||
rows.forEach((row, index) => {
|
||||
const current = totals.get(row.model_group) ?? { provider: row.provider, value: 0 };
|
||||
totals.set(row.model_group, { provider: row.provider, value: current.value + values[index] });
|
||||
});
|
||||
const grand = [...totals.values()].reduce((sum, entry) => sum + entry.value, 0);
|
||||
return { totals, grand };
|
||||
};
|
||||
|
||||
const halfShares = (daily: DailyMetric[], metric: Metric, range: DateRange) => {
|
||||
const midpoint = isoDay(toDay(range.start) + Math.floor((toDay(range.end) - toDay(range.start)) / 2 + DAY_MS / 2));
|
||||
const share = (rows: DailyMetric[]) => {
|
||||
const { totals, grand } = shareByModel(
|
||||
rows,
|
||||
rows.map((row) => metricValue(row, metric)),
|
||||
);
|
||||
return {
|
||||
hasUsage: grand > 0,
|
||||
of: (model: string) => (grand === 0 ? 0 : ((totals.get(model)?.value ?? 0) / grand) * 100),
|
||||
};
|
||||
};
|
||||
return {
|
||||
earlier: share(daily.filter((row) => row.date < midpoint)),
|
||||
later: share(daily.filter((row) => row.date >= midpoint)),
|
||||
};
|
||||
};
|
||||
|
||||
export const rankModels = (
|
||||
rows: ModelMetric[],
|
||||
daily: DailyMetric[],
|
||||
metric: Metric,
|
||||
range: DateRange,
|
||||
): RankedModel[] => {
|
||||
const { totals, grand } = shareByModel(
|
||||
rows,
|
||||
rows.map((row) => metricValue(row, metric)),
|
||||
);
|
||||
const { earlier, later } = halfShares(daily, metric, range);
|
||||
const comparable = earlier.hasUsage && later.hasUsage;
|
||||
return [...totals.entries()]
|
||||
.sort((a, b) => b[1].value - a[1].value)
|
||||
.map(([model_group, entry]) => ({
|
||||
model_group,
|
||||
provider: entry.provider,
|
||||
share: grand === 0 ? 0 : (entry.value / grand) * 100,
|
||||
delta: comparable ? later.of(model_group) - earlier.of(model_group) : 0,
|
||||
}));
|
||||
};
|
||||
|
|
@ -1,9 +1,16 @@
|
|||
"use client";
|
||||
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import ModelInsightsView from "./_components/ModelInsightsView";
|
||||
import { useEffect } from "react";
|
||||
import { useRouter } from "next/navigation";
|
||||
import LoadingScreen from "@/components/common_components/LoadingScreen";
|
||||
import { uiHref } from "@/utils/uiHref";
|
||||
|
||||
export default function ModelInsightsPage() {
|
||||
const { accessToken } = useAuthorized();
|
||||
return <ModelInsightsView accessToken={accessToken} />;
|
||||
const router = useRouter();
|
||||
|
||||
useEffect(() => {
|
||||
router.replace(`${uiHref("usage")}${window.location.search}${window.location.hash}`);
|
||||
}, [router]);
|
||||
|
||||
return <LoadingScreen />;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,5 @@
|
|||
"use client";
|
||||
|
||||
/**
|
||||
* Usage overview: a stat strip, the Model Leaderboard's stacked chart and
|
||||
* tooltip applied to the usage aggregate, then a ranked share list in
|
||||
* matching colors.
|
||||
*/
|
||||
|
||||
import { type ReactNode } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import type { DailyData } from "@/components/UsagePage/types";
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@ import {
|
|||
ShieldCheck,
|
||||
Tags,
|
||||
Terminal,
|
||||
Trophy,
|
||||
User,
|
||||
Users,
|
||||
Receipt,
|
||||
|
|
@ -333,17 +332,6 @@ const menuGroups: MenuGroup[] = [
|
|||
icon: <Bell {...ICON} />,
|
||||
roles: rolesWithCapability("viewGlobalSpend"),
|
||||
},
|
||||
{
|
||||
key: "model-insights",
|
||||
page: "model-insights",
|
||||
icon: <Trophy {...ICON} />,
|
||||
roles: all_admin_roles,
|
||||
label: (
|
||||
<span className="flex items-center gap-2">
|
||||
<Trans i18nKey="shell.nav.items.modelLeaderboard" /> <BetaBadge />
|
||||
</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "cost-optimization",
|
||||
page: "cost-optimization",
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ export const pageDescriptions: Record<string, ParseKeys> = {
|
|||
new_usage: "shell.pageDescriptions.newUsage",
|
||||
alerts: "shell.pageDescriptions.alerts",
|
||||
"cost-optimization": "shell.pageDescriptions.costOptimization",
|
||||
"model-insights": "shell.pageDescriptions.modelInsights",
|
||||
logs: "shell.pageDescriptions.logs",
|
||||
lens: "shell.pageDescriptions.lens",
|
||||
"guardrails-monitor": "shell.pageDescriptions.guardrailsMonitor",
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import { cn } from "@/lib/cva.config";
|
|||
const LOG_SCALE_DOMAIN: [number, "auto"] = [1, "auto"];
|
||||
const LINEAR_SCALE_DOMAIN: [number, "auto"] = [0, "auto"];
|
||||
|
||||
/** The Model Leaderboard palette; shared so the usage page stacks in the same colors. */
|
||||
export const STACKED_USAGE_PALETTE = [
|
||||
"#ec4899",
|
||||
"#a855f7",
|
||||
|
|
|
|||
|
|
@ -8131,7 +8131,6 @@
|
|||
"toolPolicies": "Tool Policies",
|
||||
"usage": "Usage",
|
||||
"alerts": "Alerts",
|
||||
"modelLeaderboard": "Model Leaderboard",
|
||||
"costOptimization": "Cost Optimization",
|
||||
"logs": "Logs",
|
||||
"lens": "Lens",
|
||||
|
|
@ -8178,7 +8177,6 @@
|
|||
"newUsage": "View usage analytics and metrics",
|
||||
"alerts": "Review and resolve model deprecation alerts across keys and teams",
|
||||
"costOptimization": "Track and configure cost-saving features: prompt compression, caching, and auto routing",
|
||||
"modelInsights": "Model Leaderboard: compare usage, spend, tokens, and task mix across this gateway",
|
||||
"logs": "Access request and response logs",
|
||||
"lens": "Review agent activity and investigate patterns with supporting evidence",
|
||||
"guardrailsMonitor": "Monitor guardrail performance and view logs",
|
||||
|
|
@ -19967,38 +19965,6 @@
|
|||
"deleted": "Alias deleted successfully"
|
||||
}
|
||||
},
|
||||
"modelInsights": {
|
||||
"modelInsightsView": {
|
||||
"granularity": {
|
||||
"day": "Daily",
|
||||
"week": "Weekly"
|
||||
},
|
||||
"metrics": {
|
||||
"requests": "requests",
|
||||
"spend": "spend",
|
||||
"tokens": "tokens"
|
||||
},
|
||||
"byProvider": "by {{provider}}",
|
||||
"loadError": "Could not load model insights",
|
||||
"title": "Model Leaderboard",
|
||||
"description": "See which models your gateway used from {{start}} through {{end}}",
|
||||
"topModels": {
|
||||
"title": "Top models",
|
||||
"description": "{{granularity}} {{metric}} across your gateway"
|
||||
},
|
||||
"bucketSize": "Bucket size",
|
||||
"gatewayTotal": "Gateway total",
|
||||
"leaderboard": {
|
||||
"title": "Leaderboard",
|
||||
"description": "Share of {{metric}}, with the change between the first and second half of the period"
|
||||
},
|
||||
"costPerSession": {
|
||||
"title": "Cost per session",
|
||||
"description": "Session cost is not estimated from request counts",
|
||||
"hint": "Add a stable session_id to requests to unlock accurate session-level model comparisons in a future bounded session rollup"
|
||||
}
|
||||
}
|
||||
},
|
||||
"transformRequest": {
|
||||
"transformRequestPanel": {
|
||||
"title": "Playground",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue