scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-02 02:11:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

feat(ui): show daily token totals on the model leaderboard (#44044)

Browse source 
* feat(ui): bucket model leaderboard usage by day or week

* test(ui): cover daily buckets in model leaderboard series

* feat(ui): add daily/weekly toggle and per-bucket total to model leaderboard chart

* test(ui): cover the daily/weekly toggle on the model leaderboard

* feat(model-insights): add gateway-wide daily totals to the response type

* feat(model-insights): return per-day totals across every model, not just the top ranked ones

* test(model-insights): daily totals include models outside the top ranking

* chore(model-insights): regenerate lazy openapi snapshot for daily totals

* chore(ui): regenerate api types for model insights daily totals

* fix(ui): compute leaderboard bucket totals from gateway-wide daily totals

* fix(ui): show the gateway total, not the top-ten subtotal, in the leaderboard tooltip

* test(ui): cover gateway-wide bucket totals on the model leaderboard

* fix(model-insights): type daily totals as an immutable tuple

* fix(model-insights): build daily totals without new mutable collections
This commit is contained in:
ishaan-berri 2026-10-01 12:12:30 -07:00 • committed by GitHub
parent c030191be6
commit fcf87972fd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 248 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -41284,6 +41284,39 @@
"title": "ModelInsightDailyMetric",
"type": "object"
},
"ModelInsightDailyTotal": {
"properties": {
"completion_tokens": {
"title": "Completion Tokens",
"type": "integer"
},
"date": {
"title": "Date",
"type": "string"
},
"prompt_tokens": {
"title": "Prompt Tokens",
"type": "integer"
},
"requests": {
"title": "Requests",
"type": "integer"
},
"spend": {
"title": "Spend",
"type": "number"
}
},
"required": [
"date",
"spend",
"prompt_tokens",
"completion_tokens",
"requests"
],
"title": "ModelInsightDailyTotal",
"type": "object"
},
"ModelInsightMetric": {
"properties": {
"completion_tokens": {
@ -41415,6 +41448,13 @@
"title": "Daily",
"type": "array"
},
"daily_totals": {
"items": {
"$ref": "#/components/schemas/ModelInsightDailyTotal"
},
"title": "Daily Totals",
"type": "array"
},
"end_date": {
"title": "End Date",
"type": "string"
@ -41435,6 +41475,7 @@
"start_date",
"end_date",
"daily",
"daily_totals",
"top_models"
],
"title": "ModelInsightsResponse",

26
litellm/proxy/management_endpoints/model_insights_endpoints.py
View file

@ -14,6 +14,7 @@ from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks
from litellm.repositories.table_repositories import DailyModelUsageRepository
from litellm.types.model_insights import (
ModelInsightDailyMetric,
ModelInsightDailyTotal,
ModelInsightMetric,
ModelInsightsMetric,
ModelInsightsResponse,
@ -45,12 +46,18 @@ class _GroupedDaily(_GroupedModel):
date: str
class _GroupedDate(BaseModel):
date: str
sums: _Sums = Field(alias="_sum")
class _GroupedTask(_GroupedModel):
task_type: str
_MODEL_ROWS: Final = TypeAdapter(list[_GroupedModel])
_DAILY_ROWS: Final = TypeAdapter(list[_GroupedDaily])
_DATE_ROWS: Final = TypeAdapter(list[_GroupedDate])
_TASK_ROWS: Final = TypeAdapter(list[_GroupedTask])
_UNCATEGORIZED_TASK: Final = ModelInsightTask(
task_type=MODEL_INSIGHTS_DEFAULT_TASK, label="Uncategorized", category="General"
@ -111,6 +118,16 @@ def _daily_metric(row: _GroupedDaily) -> ModelInsightDailyMetric:
return ModelInsightDailyMetric(date=row.date, **_metric(row).model_dump())
def _daily_total(row: _GroupedDate) -> ModelInsightDailyTotal:
return ModelInsightDailyTotal(
date=row.date,
spend=row.sums.spend,
prompt_tokens=row.sums.prompt_tokens,
completion_tokens=row.sums.completion_tokens,
requests=row.sums.request_count,
)
def _summarize_tasks(rows: list[_GroupedTask], metric: ModelInsightsMetric) -> list[ModelInsightTaskSummary]:
catalog: Final = load_model_insight_tasks()
first_seen: Final = {task: index for index, task in enumerate(dict.fromkeys(row.task_type for row in rows))}
@ -193,11 +210,20 @@ async def get_model_insights(
if model_rows
else []
)
date_rows: Final = _DATE_ROWS.validate_python(
await table.group_by(
by=["date"], # mutable-ok: prisma group_by requires a list of fields
sum=_SUM_FIELDS,
where=date_window,
order={"date": "asc"}, # mutable-ok: prisma order clause must be a dict
)
)
return ModelInsightsResponse(
start_date=start_day.isoformat(),
end_date=end_day.isoformat(),
top_models=[_metric(row) for row in model_rows],
daily=[_daily_metric(row) for row in daily_rows],
daily_totals=tuple(_daily_total(row) for row in date_rows),
)

9
litellm/types/model_insights.py
View file

@ -21,6 +21,14 @@ class ModelInsightDailyMetric(ModelInsightMetric):
date: str
class ModelInsightDailyTotal(BaseModel):
date: str
spend: float
prompt_tokens: int
completion_tokens: int
requests: int
class ModelInsightTask(BaseModel):
task_type: str
label: str
@ -38,6 +46,7 @@ class ModelInsightsResponse(BaseModel):
start_date: str
end_date: str
daily: list[ModelInsightDailyMetric]
daily_totals: tuple[ModelInsightDailyTotal, ...]
top_models: list[ModelInsightMetric]

28
tests/unit/proxy/management_endpoints/test_model_insights_endpoints.py
View file

@ -45,7 +45,7 @@ def test_model_insights_reads_only_bounded_rollup() -> None:
custom_llm_provider="openai",
)
table = MagicMock()
table.group_by = AsyncMock(side_effect=[[model, prompt_heavy_model], [daily]])
table.group_by = AsyncMock(side_effect=[[model, prompt_heavy_model], [daily], []])
prisma = MagicMock()
prisma.db.litellm_dailymodelusage = table
prisma.db.query_raw = AsyncMock()
@ -60,7 +60,7 @@ def test_model_insights_reads_only_bounded_rollup() -> None:
assert response.status_code == 200
assert response.json()["top_models"][0]["model_group"] == "long-context"
assert "by_task" not in response.json()
assert table.group_by.await_count == 2
assert table.group_by.await_count == 3
prisma.db.query_raw.assert_not_awaited()
prisma.db.litellm_spendlogs.find_many.assert_not_awaited()
@ -96,11 +96,11 @@ def test_model_insights_ranks_top_models_by_selected_metric() -> None:
)
request_heavy["_sum"]["request_count"] = "500"
table = MagicMock()
table.group_by = AsyncMock(side_effect=[[token_heavy, request_heavy], []])
table.group_by = AsyncMock(side_effect=[[token_heavy, request_heavy], [], []])
by_requests = _call(table, "metric=requests").json()
by_tokens = _call(
MagicMock(group_by=AsyncMock(side_effect=[[token_heavy, request_heavy], []])), "metric=tokens"
MagicMock(group_by=AsyncMock(side_effect=[[token_heavy, request_heavy], [], []])), "metric=tokens"
).json()
assert by_requests["top_models"][0]["model_group"] == "busy"
@ -110,7 +110,7 @@ def test_model_insights_ranks_top_models_by_selected_metric() -> None:
def test_model_insights_scopes_daily_to_ranked_deployments() -> None:
ranked = _grouped_row(model_group="shared", model="m1", custom_llm_provider="openai")
table = MagicMock()
table.group_by = AsyncMock(side_effect=[[ranked], []])
table.group_by = AsyncMock(side_effect=[[ranked], [], []])
_call(table, "metric=tokens")
@ -119,6 +119,24 @@ def test_model_insights_scopes_daily_to_ranked_deployments() -> None:
assert "model_group" not in daily_where
def test_model_insights_daily_totals_cover_every_model_not_just_the_ranked_ones() -> None:
ranked = _grouped_row(model_group="ranked", model="m1", custom_llm_provider="openai")
ranked_day = _grouped_row(date="2026-09-28", model_group="ranked", model="m1", custom_llm_provider="openai")
whole_gateway_day = _grouped_row(prompt_tokens="7000", completion_tokens="3000", date="2026-09-28")
table = MagicMock()
table.group_by = AsyncMock(side_effect=[[ranked], [ranked_day], [whole_gateway_day]])
body = _call(table, "metric=tokens").json()
totals_call = table.group_by.await_args_list[2].kwargs
assert totals_call["by"] == ["date"]
assert "OR" not in totals_call["where"]
assert body["daily_totals"] == [
{"date": "2026-09-28", "spend": 1.25, "prompt_tokens": 7000, "completion_tokens": 3000, "requests": 3}
]
assert body["daily"][0]["prompt_tokens"] + body["daily"][0]["completion_tokens"] < 10000
def _task_rows() -> list[dict[str, object]]:
def row(task: str, group: str, requests: str, spend: float) -> dict[str, object]:
base = _grouped_row(task_type=task, model_group=group, model=group, custom_llm_provider="openai")

24
ui/litellm-dashboard/src/app/(dashboard)/model-insights/_components/ModelInsightsView.test.tsx
View file

@ -14,7 +14,11 @@ vi.mock("@/components/ui/chart", () => ({
}));
vi.mock("recharts", () => ({
Bar: () => null,
BarChart: ({ children }: { children: React.ReactNode }) => <div>{children}</div>,
BarChart: ({ children, data }: { children: React.ReactNode; data: { date: string }[] }) => (
<div data-testid="usage-chart" data-first={data[0]?.date} data-buckets={data.length}>
{children}
</div>
),
CartesianGrid: () => null,
Treemap: () => null,
XAxis: () => null,
@ -38,6 +42,7 @@ const response = {
end_date: "2026-09-28",
top_models: [metrics],
daily: [{ ...metrics, date: "2026-09-28" }],
daily_totals: [{ date: "2026-09-28", spend: 2.5, prompt_tokens: 1000, completion_tokens: 2000, requests: 12 }],
};
const taskResponse = {
@ -145,4 +150,21 @@ describe("ModelInsightsView", () => {
await screen.findByText("Share of spend, with the change between the first and second half of the period"),
).toBeInTheDocument();
});
it("charts one bar per day by default and switches to weekly bars", async () => {
render(<ModelInsightsView accessToken="token" />);
await screen.findByText("fast-chat");
const chart = screen.getByTestId("usage-chart");
const days = (Date.parse(response.end_date) - Date.parse(response.start_date)) / 86_400_000 + 1;
expect(screen.getByRole("tab", { name: "Daily" })).toHaveAttribute("aria-selected", "true");
expect(chart).toHaveAttribute("data-buckets", String(days));
expect(screen.getByText("Daily tokens across your gateway")).toBeInTheDocument();
await userEvent.click(screen.getByRole("tab", { name: "Weekly" }));
expect(chart).toHaveAttribute("data-buckets", String(Math.ceil(days / 7)));
expect(chart).toHaveAttribute("data-first", response.start_date);
expect(screen.getByText("Weekly tokens across your gateway")).toBeInTheDocument();
});
});

38
ui/litellm-dashboard/src/app/(dashboard)/model-insights/_components/ModelInsightsView.tsx
View file

@ -15,8 +15,10 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@
import { Skeleton } from "@/components/ui/skeleton";
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
import {
buildWeeklySeries,
buildBucketTotals,
buildSeries,
formatMetric,
Granularity,
Metric,
ModelInsightsResponse,
ModelInsightTasksResponse,
@ -46,6 +48,8 @@ const CATEGORY_COLORS: Record<string, string> = {
Data: "#3b82f6",
};
const SCALES = ["linear", "log"] as const;
const GRANULARITIES = ["day", "week"] as const;
const GRANULARITY_LABELS: Record<Granularity, string> = { day: "Daily", week: "Weekly" };
const METRIC_LABELS: Record<Metric, string> = { requests: "requests", spend: "spend", tokens: "tokens" };
const RANKING_ROWS = 5;
@ -112,6 +116,7 @@ export default function ModelInsightsView({ accessToken }: { accessToken: string
const [loaded, setLoaded] = React.useState<{ metric: Metric; response: ModelInsightsResponse } | null>(null);
const [metric, setMetric] = React.useState<Metric>("tokens");
const [scale, setScale] = React.useState<Scale>("linear");
const [granularity, setGranularity] = React.useState<Granularity>("day");
const [taskMetric, setTaskMetric] = React.useState<Metric>("spend");
const [taskData, setTaskData] = React.useState<ModelInsightTasksResponse | null>(null);
const [taskError, setTaskError] = React.useState<string | null>(null);
@ -159,8 +164,12 @@ export default function ModelInsightsView({ accessToken }: { accessToken: string
const range = React.useMemo(() => ({ start: data?.start_date ?? "", end: data?.end_date ?? "" }), [data]);
const models = React.useMemo(() => (data ? modelOrder(data.daily, shown) : []), [data, shown]);
const series = React.useMemo(
() => (data ? buildWeeklySeries(data.daily, models, shown, range) : []),
[data, models, shown, range],
() => (data ? buildSeries(data.daily, models, shown, { ...range, granularity }) : []),
[data, models, shown, range, granularity],
);
const bucketTotals = React.useMemo(
() => (data ? buildBucketTotals(data.daily_totals, shown, { ...range, granularity }) : new Map<string, number>()),
[data, shown, range, granularity],
);
const ranking = React.useMemo(
() => (data ? rankModels(data.top_models, data.daily, shown, range) : []),
@ -212,7 +221,9 @@ export default function ModelInsightsView({ accessToken }: { accessToken: string
<CardHeader className="flex-row items-start justify-between space-y-0">
<div>
<CardTitle>Top models</CardTitle>
<CardDescription>Weekly {METRIC_LABELS[shown]} across your gateway</CardDescription>
<CardDescription>
{GRANULARITY_LABELS[granularity]} {METRIC_LABELS[shown]} across your gateway
</CardDescription>
</div>
<div className="flex items-center gap-3">
<Tabs value={metric} onValueChange={(value) => setMetric(value as Metric)}>
@ -224,6 +235,15 @@ export default function ModelInsightsView({ accessToken }: { accessToken: string
))}
</TabsList>
</Tabs>
<Tabs value={granularity} onValueChange={(value) => setGranularity(value as Granularity)}>
<TabsList aria-label="Bucket size">
{GRANULARITIES.map((value) => (
<TabsTrigger key={value} value={value}>
{GRANULARITY_LABELS[value]}
</TabsTrigger>
))}
</TabsList>
</Tabs>
<Tabs value={scale} onValueChange={(value) => setScale(value as Scale)}>
<TabsList>
{SCALES.map((value) => (
@ -248,7 +268,15 @@ export default function ModelInsightsView({ accessToken }: { accessToken: string
axisLine={false}
tickFormatter={(value) => formatMetric(Number(value), shown)}
/>
<ChartTooltip content={<ChartTooltipContent />} />
<ChartTooltip
content={
<ChartTooltipContent
labelFormatter={(label) =>
`${label} · Gateway total ${formatMetric(bucketTotals.get(String(label)) ?? 0, shown)}`
}
/>
}
/>
{models.map((model, index) => (
<Bar
key={model}

54
ui/litellm-dashboard/src/app/(dashboard)/model-insights/_components/modelInsightsData.test.ts
View file

@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";
import { buildWeeklySeries, DailyMetric, formatMetric, modelOrder, rankModels } from "./modelInsightsData";
import { buildBucketTotals, buildSeries, DailyMetric, formatMetric, modelOrder, rankModels } from "./modelInsightsData";
const row = (over: Partial<DailyMetric>): DailyMetric => ({
model_group: "a",
@ -16,7 +16,7 @@ const row = (over: Partial<DailyMetric>): DailyMetric => ({
...over,
});
describe("buildWeeklySeries", () => {
describe("buildSeries", () => {
const range = { start: "2026-01-01", end: "2026-01-15" };
it("sums days into 7-day buckets per model", () => {
@ -26,7 +26,7 @@ describe("buildWeeklySeries", () => {
row({ date: "2026-01-08", requests: 4 }),
row({ date: "2026-01-02", model_group: "b", requests: 8 }),
];
expect(buildWeeklySeries(rows, ["a", "b"], "requests", range)).toEqual([
expect(buildSeries(rows, ["a", "b"], "requests", { ...range, granularity: "week" })).toEqual([
{ date: "2026-01-01", a: 3, b: 8 },
{ date: "2026-01-08", a: 4, b: 0 },
{ date: "2026-01-15", a: 0, b: 0 },
@ -35,12 +35,58 @@ describe("buildWeeklySeries", () => {
it("keeps weeks with no usage as zero instead of dropping them", () => {
const rows = [row({ date: "2026-01-01", requests: 1 }), row({ date: "2026-01-15", requests: 2 })];
expect(buildWeeklySeries(rows, ["a"], "requests", range).map((week) => [week.date, week.a])).toEqual([
expect(
buildSeries(rows, ["a"], "requests", { ...range, granularity: "week" }).map((week) => [week.date, week.a]),
).toEqual([
["2026-01-01", 1],
["2026-01-08", 0],
["2026-01-15", 2],
]);
});
it("gives every day its own bucket with that day's token total", () => {
const rows = [
row({ date: "2026-01-01", prompt_tokens: 100, completion_tokens: 50 }),
row({ date: "2026-01-01", prompt_tokens: 10, completion_tokens: 5 }),
row({ date: "2026-01-03", prompt_tokens: 7, completion_tokens: 3 }),
];
const daily = buildSeries(rows, ["a"], "tokens", { start: "2026-01-01", end: "2026-01-03", granularity: "day" });
expect(daily).toEqual([
{ date: "2026-01-01", a: 165 },
{ date: "2026-01-02", a: 0 },
{ date: "2026-01-03", a: 10 },
]);
});
});
describe("buildBucketTotals", () => {
const total = (date: string, prompt_tokens: number) => ({
date,
spend: 0,
prompt_tokens,
completion_tokens: 1,
requests: 0,
});
const totals = [total("2026-01-01", 9), total("2026-01-03", 4), total("2026-01-08", 99)];
it("keys each day's gateway-wide total by its own date", () => {
const daily = buildBucketTotals(totals, "tokens", { start: "2026-01-01", end: "2026-01-08", granularity: "day" });
expect([...daily]).toEqual([
["2026-01-01", 10],
["2026-01-03", 5],
["2026-01-08", 100],
]);
});
it("sums days into the same week start used by the chart's x-axis", () => {
const window = { start: "2026-01-01", end: "2026-01-08", granularity: "week" } as const;
const weekly = buildBucketTotals(totals, "tokens", window);
expect([...weekly]).toEqual([
["2026-01-01", 15],
["2026-01-08", 100],
]);
expect([...weekly.keys()]).toEqual(buildSeries([], [], "tokens", window).map((bucket) => bucket.date));
});
});
describe("modelOrder", () => {

38
ui/litellm-dashboard/src/app/(dashboard)/model-insights/_components/modelInsightsData.ts
View file

@ -12,10 +12,13 @@ export type ModelMetric = {
failed_requests: number;
};
export type DailyMetric = ModelMetric & { date: string };
type Usage = Pick<ModelMetric, "spend" | "prompt_tokens" | "completion_tokens" | "requests">;
export type DailyTotal = Usage & { date: string };
export type ModelInsightsResponse = {
start_date: string;
end_date: string;
daily: DailyMetric[];
daily_totals: DailyTotal[];
top_models: ModelMetric[];
};
export type TaskSummary = {
@ -30,10 +33,12 @@ export type TaskSummary = {
export type ModelInsightTasksResponse = { start_date: string; end_date: string; tasks: TaskSummary[] };
export type RankedModel = { model_group: string; provider: string; share: number; delta: number };
const DAY_MS = 86_400_000;
const WEEK_DAYS = 7;
export type Granularity = "day" | "week";
export const metricValue = (row: ModelMetric, metric: Metric) => {
const DAY_MS = 86_400_000;
const BUCKET_DAYS: Record<Granularity, number> = { day: 1, week: 7 };
export const metricValue = (row: Usage, metric: Metric) => {
if (metric === "requests") return row.requests;
if (metric === "spend") return row.spend;
return row.prompt_tokens + row.completion_tokens;
@ -66,21 +71,34 @@ export const modelOrder = (rows: DailyMetric[], metric: Metric) => {
return [...totals.entries()].sort((a, b) => b[1] - a[1]).map(([model]) => model);
};
export const buildWeeklySeries = (rows: DailyMetric[], models: string[], metric: Metric, range: DateRange) => {
const weekMs = WEEK_DAYS * DAY_MS;
const origin = toDay(range.start);
const weekCount = Math.floor((toDay(range.end) - origin) / weekMs) + 1;
const buckets = Array.from({ length: weekCount }, (_, week) => ({
date: isoDay(origin + week * weekMs),
export type SeriesWindow = DateRange & { granularity: Granularity };
export const buildSeries = (rows: DailyMetric[], models: string[], metric: Metric, window: SeriesWindow) => {
const bucketMs = BUCKET_DAYS[window.granularity] * DAY_MS;
const origin = toDay(window.start);
const bucketCount = Math.floor((toDay(window.end) - origin) / bucketMs) + 1;
const buckets = Array.from({ length: bucketCount }, (_, index) => ({
date: isoDay(origin + index * bucketMs),
...Object.fromEntries(models.map((model) => [model, 0])),
})) as Record<string, number | string>[];
for (const row of rows) {
const bucket = buckets[Math.floor((toDay(row.date) - origin) / weekMs)];
const bucket = buckets[Math.floor((toDay(row.date) - origin) / bucketMs)];
if (bucket) bucket[row.model_group] = Number(bucket[row.model_group] ?? 0) + metricValue(row, metric);
}
return buckets;
};
export const buildBucketTotals = (totals: DailyTotal[], metric: Metric, window: SeriesWindow) => {
const bucketMs = BUCKET_DAYS[window.granularity] * DAY_MS;
const origin = toDay(window.start);
const byBucket = new Map<string, number>();
for (const row of totals) {
const bucket = isoDay(origin + Math.floor((toDay(row.date) - origin) / bucketMs) * bucketMs);
byBucket.set(bucket, (byBucket.get(bucket) ?? 0) + metricValue(row, metric));
}
return byBucket;
};
const shareByModel = (rows: { model_group: string; provider: string }[], values: number[]) => {
const totals = new Map<string, { provider: string; value: number }>();
rows.forEach((row, index) => {

15
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -37160,6 +37160,19 @@ export interface components {
/** Successful Requests */
successful_requests: number;
};
/** ModelInsightDailyTotal */
ModelInsightDailyTotal: {
/** Completion Tokens */
completion_tokens: number;
/** Date */
date: string;
/** Prompt Tokens */
prompt_tokens: number;
/** Requests */
requests: number;
/** Spend */
spend: number;
};
/** ModelInsightMetric */
ModelInsightMetric: {
/** Completion Tokens */
@ -37211,6 +37224,8 @@ export interface components {
ModelInsightsResponse: {
/** Daily */
daily: components["schemas"]["ModelInsightDailyMetric"][];
/** Daily Totals */
daily_totals: components["schemas"]["ModelInsightDailyTotal"][];
/** End Date */
end_date: string;
/** Start Date */