scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

feat(usage): show smart router model breakdowns

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Krrish Dholakia 2026-07-16 07:17:22 +00:00
parent bf3a058781
commit ac827e669a
11 changed files with 345 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -133,6 +133,13 @@ def update_breakdown_metrics(
breakdown.model_groups[record.model_group].metrics = update_metrics(
breakdown.model_groups[record.model_group].metrics, record
)
if record.model:
if record.model not in breakdown.model_groups[record.model_group].model_breakdown:
breakdown.model_groups[record.model_group].model_breakdown[record.model] = SpendMetrics()
breakdown.model_groups[record.model_group].model_breakdown[record.model] = update_metrics(
breakdown.model_groups[record.model_group].model_breakdown[record.model],
record,
)
# Update API key breakdown for this model
if record.api_key not in breakdown.model_groups[record.model_group].api_key_breakdown:
@ -484,6 +491,7 @@ def _build_aggregated_sql_query(
(date, model),
(date, model, api_key),
(date, model_group),
(date, model, model_group),
(date, model_group, api_key),
(date, custom_llm_provider),
(date, custom_llm_provider, api_key),
@ -588,6 +596,7 @@ _GROUP_DATE_API_KEY = 31 # 0b0011111
_GROUP_DATE_MODEL = 47 # 0b0101111
_GROUP_DATE_MODEL_API_KEY = 15 # 0b0001111
_GROUP_DATE_MODEL_GROUP = 55 # 0b0110111
_GROUP_DATE_MODEL_MODEL_GROUP = 39 # 0b0100111
_GROUP_DATE_MODEL_GROUP_API_KEY = 23 # 0b0010111
_GROUP_DATE_PROVIDER = 59 # 0b0111011
_GROUP_DATE_PROVIDER_API_KEY = 27 # 0b0011011
@ -695,6 +704,13 @@ def _aggregate_grouping_sets_records_sync(
elif level == _GROUP_DATE_MODEL_GROUP:
if record.model_group:
assign_metric_with_metadata(breakdown.model_groups, record.model_group, metrics)
elif level == _GROUP_DATE_MODEL_MODEL_GROUP:
if record.model_group and record.model:
parent = breakdown.model_groups.get(record.model_group)
if parent is None:
parent = MetricWithMetadata(metrics=SpendMetrics(), metadata={})
breakdown.model_groups[record.model_group] = parent
parent.model_breakdown[record.model] = metrics
elif level == _GROUP_DATE_MODEL_GROUP_API_KEY:
if record.model_group and record.api_key:
assign_api_key_breakdown(

1
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -49,6 +49,7 @@ class MetricWithMetadata(MetricBase):
metadata: Dict[str, Any] = Field(default_factory=dict)
# API key breakdown for this metric (e.g., which API keys are using this MCP server)
api_key_breakdown: Dict[str, KeyMetricWithMetadata] = Field(default_factory=dict) # api_key -> {metrics, metadata}
model_breakdown: Dict[str, SpendMetrics] = Field(default_factory=dict)
class BreakdownMetrics(BaseModel):

86
tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
View file

@ -11,15 +11,20 @@ sys.path.insert(
from litellm.proxy.management_endpoints.common_daily_activity import (
_adjust_dates_for_timezone,
_aggregate_grouping_sets_records_sync,
_build_aggregated_sql_query,
_is_user_agent_tag,
_record_to_spend_metrics,
get_api_key_metadata,
get_daily_activity,
get_daily_activity_aggregated,
update_breakdown_metrics,
update_metrics,
)
from litellm.types.proxy.management_endpoints.common_daily_activity import SpendMetrics
from litellm.types.proxy.management_endpoints.common_daily_activity import (
BreakdownMetrics,
SpendMetrics,
)
@pytest.mark.asyncio
@ -837,6 +842,85 @@ class TestBuildAggregatedSqlQuery:
assert "model = $4" in sql
assert "api_key = $5" in sql
def test_includes_model_breakdown_for_model_groups(self):
sql, _ = _build_aggregated_sql_query(
table_name="litellm_dailyuserspend",
entity_id_field="user_id",
entity_id=None,
start_date="2026-05-29",
end_date="2026-05-29",
model=None,
api_key=None,
)
assert "(date, model, model_group)" in sql
def test_aggregate_grouping_sets_populates_model_group_model_breakdown():
record = SimpleNamespace(
group_level=39,
date="2026-05-29",
api_key=None,
model="bedrock/claude-opus-4-8",
model_group="smart-router",
custom_llm_provider=None,
mcp_namespaced_tool_name=None,
endpoint=None,
spend=12.5,
prompt_tokens=8000,
completion_tokens=2000,
cache_read_input_tokens=4000,
cache_creation_input_tokens=1000,
api_requests=8,
successful_requests=8,
failed_requests=0,
)
result = _aggregate_grouping_sets_records_sync(records=[record], api_key_metadata={})
metrics = result["results"][0].breakdown.model_groups["smart-router"].model_breakdown[
"bedrock/claude-opus-4-8"
]
assert metrics.api_requests == 8
assert metrics.total_tokens == 10000
assert metrics.cache_read_input_tokens == 4000
assert metrics.cache_creation_input_tokens == 1000
def test_update_breakdown_metrics_populates_model_group_model_breakdown():
record = SimpleNamespace(
model="bedrock/claude-sonnet-4-5",
model_group="smart-router",
api_key="key-hash",
custom_llm_provider="bedrock",
mcp_namespaced_tool_name=None,
endpoint=None,
spend=4.5,
prompt_tokens=3000,
completion_tokens=1000,
cache_read_input_tokens=1500,
cache_creation_input_tokens=500,
api_requests=2,
successful_requests=2,
failed_requests=0,
)
result = update_breakdown_metrics(
breakdown=BreakdownMetrics(),
record=record,
model_metadata={},
provider_metadata={},
api_key_metadata={},
)
metrics = result.model_groups["smart-router"].model_breakdown[
"bedrock/claude-sonnet-4-5"
]
assert metrics.api_requests == 2
assert metrics.total_tokens == 4000
assert metrics.cache_read_input_tokens == 1500
assert metrics.cache_creation_input_tokens == 500
@pytest.mark.asyncio
async def test_get_daily_activity_aggregated_empty_result_set():

1
ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx
View file

@ -1157,6 +1157,7 @@ describe("UsagePage", () => {
expect(screen.getByText("Cost")).toBeInTheDocument();
expect(screen.getByText("Model Activity")).toBeInTheDocument();
expect(screen.getByText("Public Model Activity")).toBeInTheDocument();
expect(screen.getByText("Key Activity")).toBeInTheDocument();
expect(screen.getByText("MCP Server Activity")).toBeInTheDocument();
expect(screen.getByText("Endpoint Activity")).toBeInTheDocument();

8
ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx
View file

@ -439,6 +439,10 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
[userSpendData.results],
);
const modelMetrics = useMemo(() => processActivityData(userSpendData, "models", teams), [userSpendData, teams]);
const modelGroupMetrics = useMemo(
() => processActivityData(userSpendData, "model_groups", teams),
[userSpendData, teams],
);
const keyMetrics = useMemo(() => processActivityData(userSpendData, "api_keys", teams), [userSpendData, teams]);
const mcpServerMetrics = useMemo(
() => processActivityData(userSpendData, "mcp_servers", teams),
@ -534,6 +538,7 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
<TabList variant="solid" className="mt-1">
<Tab>Cost</Tab>
<Tab>Model Activity</Tab>
<Tab>Public Model Activity</Tab>
<Tab>Key Activity</Tab>
<Tab>MCP Server Activity</Tab>
<Tab>Endpoint Activity</Tab>
@ -841,6 +846,9 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
<TabPanel>
<ActivityMetrics modelMetrics={modelMetrics} />
</TabPanel>
<TabPanel>
<ActivityMetrics modelMetrics={modelGroupMetrics} />
</TabPanel>
<TabPanel>
<ActivityMetrics modelMetrics={keyMetrics} />
</TabPanel>

75
ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.test.tsx
View file

@ -1,4 +1,4 @@
import { render, screen } from "@testing-library/react";
import { render, screen, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { describe, expect, it } from "vitest";
import KeyModelUsageView from "./KeyModelUsageView";
@ -13,6 +13,9 @@ describe("KeyModelUsageView", () => {
successful_requests: 100,
failed_requests: 5,
tokens: 50000,
request_share: 34.4,
cache_read_input_tokens: 20000,
cache_creation_input_tokens: 10000,
},
{
model: "gpt-3.5-turbo",
@ -21,6 +24,9 @@ describe("KeyModelUsageView", () => {
successful_requests: 195,
failed_requests: 5,
tokens: 100000,
request_share: 65.6,
cache_read_input_tokens: 40000,
cache_creation_input_tokens: 20000,
},
];
@ -59,9 +65,12 @@ describe("KeyModelUsageView", () => {
render(<KeyModelUsageView topModels={mockTopModels} />);
expect(screen.getByText("Model")).toBeInTheDocument();
expect(screen.getByText("Spend (USD)")).toBeInTheDocument();
expect(screen.getByText("Requests")).toBeInTheDocument();
expect(screen.getByText("Total Tokens")).toBeInTheDocument();
expect(screen.getByText("Successful")).toBeInTheDocument();
expect(screen.getByText("Failed")).toBeInTheDocument();
expect(screen.getByText("Tokens")).toBeInTheDocument();
expect(screen.getByText("Cache Read")).toBeInTheDocument();
expect(screen.getByText("Cache Write")).toBeInTheDocument();
});
it("should display model data in table view", () => {
@ -102,6 +111,12 @@ describe("KeyModelUsageView", () => {
expect(screen.getByText("$1,234,567.89")).toBeInTheDocument();
});
it("should display request counts and request share", () => {
render(<KeyModelUsageView topModels={mockTopModels} />);
expect(screen.getByText("105 (34.4%)")).toBeInTheDocument();
expect(screen.getByText("200 (65.6%)")).toBeInTheDocument();
});
it("should display successful requests with green styling", () => {
render(<KeyModelUsageView topModels={mockTopModels} />);
const successfulElements = screen.getAllByText("100");
@ -109,6 +124,13 @@ describe("KeyModelUsageView", () => {
expect(greenElement).toBeDefined();
});
it("should display prompt cache tokens", () => {
render(<KeyModelUsageView topModels={mockTopModels} />);
expect(screen.getAllByText("20,000")).toHaveLength(2);
expect(screen.getByText("10,000")).toBeInTheDocument();
expect(screen.getByText("40,000")).toBeInTheDocument();
});
it("should display failed requests with red styling", () => {
render(<KeyModelUsageView topModels={mockTopModels} />);
const failedElements = screen.getAllByText("5");
@ -233,6 +255,22 @@ describe("KeyModelUsageView", () => {
expect(screen.getByText("model-10")).toBeInTheDocument();
});
it("should format request counts with toLocaleString", () => {
const modelsWithLargeNumbers: TopModelData[] = [
{
model: "test-model",
spend: 100,
requests: 999999,
successful_requests: 999999,
failed_requests: 1,
tokens: 1000,
request_share: 100,
},
];
render(<KeyModelUsageView topModels={modelsWithLargeNumbers} />);
expect(screen.getByText("999,999 (100.0%)")).toBeInTheDocument();
});
it("should format successful requests with toLocaleString", () => {
const modelsWithLargeNumbers: TopModelData[] = [
{
@ -263,6 +301,39 @@ describe("KeyModelUsageView", () => {
expect(screen.getByText("999,999")).toBeInTheDocument();
});
it("should display zero percent for missing request_share", () => {
const modelsWithMissingFields: TopModelData[] = [
{
model: "test-model",
spend: 100,
requests: 10,
successful_requests: 10,
failed_requests: 0,
tokens: 1000,
},
];
render(<KeyModelUsageView topModels={modelsWithMissingFields} />);
expect(screen.getByText("10 (0.0%)")).toBeInTheDocument();
});
it("should display zero for missing cache token fields", () => {
const modelsWithMissingFields: TopModelData[] = [
{
model: "test-model",
spend: 100,
requests: 10,
successful_requests: 10,
failed_requests: 0,
tokens: 1000,
},
];
render(<KeyModelUsageView topModels={modelsWithMissingFields} />);
const row = screen.getByText("test-model").closest("tr");
const cells = within(row as HTMLTableRowElement).getAllByRole("cell");
expect(cells[6]).toHaveTextContent("0");
expect(cells[7]).toHaveTextContent("0");
});
it("should display zero for missing successful_requests", () => {
const modelsWithMissingFields: TopModelData[] = [
{

34
ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx
View file

@ -28,6 +28,22 @@ const columns: ColumnsType<TopModelData> = [
key: "spend",
render: (value) => <MoneyCell value={value} decimals={2} />,
},
{
title: "Requests",
dataIndex: "requests",
key: "requests",
render: (value, record) => (
<span>
{value?.toLocaleString() || 0} ({record.request_share?.toFixed(1) || "0.0"}%)
</span>
),
},
{
title: "Total Tokens",
dataIndex: "tokens",
key: "tokens",
render: (value) => value?.toLocaleString() || 0,
},
{
title: "Successful",
dataIndex: "successful_requests",
@ -41,9 +57,15 @@ const columns: ColumnsType<TopModelData> = [
render: (value) => <span className="text-red-600">{value?.toLocaleString() || 0}</span>,
},
{
title: "Tokens",
dataIndex: "tokens",
key: "tokens",
title: "Cache Read",
dataIndex: "cache_read_input_tokens",
key: "cache_read_input_tokens",
render: (value) => value?.toLocaleString() || 0,
},
{
title: "Cache Write",
dataIndex: "cache_creation_input_tokens",
key: "cache_creation_input_tokens",
render: (value) => value?.toLocaleString() || 0,
},
];
@ -81,11 +103,11 @@ const KeyModelUsageView: React.FC<KeyModelUsageViewProps> = ({ topModels }) => {
<div className="max-h-[234px] overflow-y-auto">
<BarChart
style={{ height: topModels.length * 40 }}
data={topModels.map((m) => ({ key: m.model, spend: m.spend }))}
data={topModels.map((model) => ({ key: model.model, requests: model.requests }))}
index="key"
categories={["spend"]}
categories={["requests"]}
colors={["cyan"]}
valueFormatter={(value) => `$${formatNumberWithCommas(value, 2)}`}
valueFormatter={(value) => formatNumberWithCommas(value, 0)}
layout="vertical"
yAxisWidth={180}
tickGap={5}

4
ui/litellm-dashboard/src/components/UsagePage/types.ts
View file

@ -30,6 +30,7 @@ export interface MetricWithMetadata {
metrics: SpendMetrics;
metadata: object;
api_key_breakdown: { [key: string]: KeyMetricWithMetadata };
model_breakdown?: { [key: string]: SpendMetrics };
}
export interface KeyMetricWithMetadata {
@ -56,9 +57,12 @@ export interface TopModelData {
model: string;
spend: number;
requests: number;
request_share?: number;
successful_requests: number;
failed_requests: number;
tokens: number;
cache_read_input_tokens?: number;
cache_creation_input_tokens?: number;
}
export interface ModelActivityData {

69
ui/litellm-dashboard/src/components/activity_metrics.test.tsx
View file

@ -1195,8 +1195,8 @@ describe("processActivityData", () => {
api_requests: 60,
successful_requests: 57,
failed_requests: 3,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 6000,
cache_creation_input_tokens: 3000,
},
metadata: { key_alias: "key-alias-1", team_id: "team1" },
},
@ -1234,6 +1234,71 @@ describe("processActivityData", () => {
expect(result["api-key-hash-1"].top_models[0].model).toBe("gpt-4");
expect(result["api-key-hash-1"].top_models[0].spend).toBe(60.0);
expect(result["api-key-hash-1"].top_models[0].requests).toBe(60);
expect(result["api-key-hash-1"].top_models[0].request_share).toBe(100);
expect(result["api-key-hash-1"].top_models[0].cache_read_input_tokens).toBe(6000);
expect(result["api-key-hash-1"].top_models[0].cache_creation_input_tokens).toBe(3000);
});
it("should populate selected model usage for public model groups", () => {
const dailyActivity: { results: DailyData[] } = {
results: [
{
date: "2025-01-01",
metrics: EMPTY_SPEND_METRICS,
breakdown: {
...EMPTY_BREAKDOWN,
model_groups: {
"smart-router": {
metrics: {
...EMPTY_SPEND_METRICS,
api_requests: 10,
total_tokens: 11000,
cache_read_input_tokens: 4000,
cache_creation_input_tokens: 2000,
},
metadata: {},
api_key_breakdown: {},
model_breakdown: {
"bedrock/claude-opus-4-8": {
...EMPTY_SPEND_METRICS,
api_requests: 8,
total_tokens: 8000,
cache_read_input_tokens: 3000,
cache_creation_input_tokens: 1500,
},
"bedrock/claude-sonnet-4-5": {
...EMPTY_SPEND_METRICS,
api_requests: 2,
total_tokens: 3000,
cache_read_input_tokens: 1000,
cache_creation_input_tokens: 500,
},
},
},
},
},
},
],
};
const result = processActivityData(dailyActivity, "model_groups");
const expectedOpusMetrics = {
model: "bedrock/claude-opus-4-8",
requests: 8,
request_share: 80,
tokens: 8000,
cache_read_input_tokens: 3000,
cache_creation_input_tokens: 1500,
};
expect(result["smart-router"].top_models).toEqual([
expect.objectContaining(expectedOpusMetrics),
expect.objectContaining({
model: "bedrock/claude-sonnet-4-5",
requests: 2,
request_share: 20,
}),
]);
});
it("should not process api_key_breakdown when key is api_keys", () => {

92
ui/litellm-dashboard/src/components/activity_metrics.tsx
View file

@ -6,7 +6,14 @@ import { Collapse } from "antd";
import React from "react";
import { Team } from "./key_team_helpers/key_list";
import KeyModelUsageView from "./UsagePage/components/KeyModelUsageView";
import { DailyData, KeyMetricWithMetadata, ModelActivityData, TopApiKeyData, TopModelData } from "./UsagePage/types";
import {
DailyData,
KeyMetricWithMetadata,
ModelActivityData,
SpendMetrics,
TopApiKeyData,
TopModelData,
} from "./UsagePage/types";
import { valueFormatter } from "./UsagePage/utils/value_formatters";
interface ActivityMetricsProps {
@ -362,7 +369,7 @@ export const formatKeyLabel = (modelData: KeyMetricWithMetadata, model: string,
// Process data function
export const processActivityData = (
dailyActivity: { results: DailyData[] },
key: "models" | "api_keys" | "mcp_servers" | "entities",
key: "models" | "model_groups" | "api_keys" | "mcp_servers" | "entities",
teams: Team[] = [],
): Record<string, ModelActivityData> => {
const modelMetrics: Record<string, ModelActivityData> = {};
@ -455,41 +462,58 @@ export const processActivityData = (
});
}
// Process Model breakdowns for each API key (only when key is 'api_keys')
if (key === "api_keys") {
Object.entries(modelMetrics).forEach(([apiKeyHash, _]) => {
const modelBreakdown: Record<string, TopModelData> = {};
if (key === "api_keys" || key === "model_groups") {
Object.entries(modelMetrics).forEach(([metricName]) => {
const modelMetricsByDay = dailyActivity.results.flatMap((day) => {
const perModelMetrics: Record<string, SpendMetrics> =
key === "api_keys"
? Object.fromEntries(
Object.entries(day.breakdown.models || {}).flatMap(([modelName, modelData]) => {
const keyData = modelData.api_key_breakdown?.[metricName];
return keyData ? [[modelName, keyData.metrics]] : [];
}),
)
: day.breakdown.model_groups?.[metricName]?.model_breakdown || {};
// Aggregate Model data for this key across all days
// We need to look in breakdown.models[model].api_key_breakdown[apiKeyHash]
dailyActivity.results.forEach((day) => {
Object.entries(day.breakdown.models || {}).forEach(([modelName, modelData]) => {
if (modelData && "api_key_breakdown" in modelData) {
const keyDataForModel = modelData.api_key_breakdown?.[apiKeyHash];
if (keyDataForModel) {
if (!modelBreakdown[modelName]) {
modelBreakdown[modelName] = {
model: modelName,
spend: 0,
requests: 0,
successful_requests: 0,
failed_requests: 0,
tokens: 0,
};
}
modelBreakdown[modelName].spend += keyDataForModel.metrics.spend;
modelBreakdown[modelName].requests += keyDataForModel.metrics.api_requests;
modelBreakdown[modelName].successful_requests += keyDataForModel.metrics.successful_requests || 0;
modelBreakdown[modelName].failed_requests += keyDataForModel.metrics.failed_requests || 0;
modelBreakdown[modelName].tokens += keyDataForModel.metrics.total_tokens;
}
}
});
return Object.entries(perModelMetrics);
});
const modelBreakdown = modelMetricsByDay.reduce<Record<string, TopModelData>>((totals, [modelName, metrics]) => {
const current = totals[modelName] ?? {
model: modelName,
spend: 0,
requests: 0,
request_share: 0,
successful_requests: 0,
failed_requests: 0,
tokens: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
};
// Sort by spend
modelMetrics[apiKeyHash].top_models = Object.values(modelBreakdown).sort((a, b) => b.spend - a.spend);
return {
...totals,
[modelName]: {
model: modelName,
spend: current.spend + metrics.spend,
requests: current.requests + metrics.api_requests,
request_share: 0,
successful_requests: current.successful_requests + (metrics.successful_requests || 0),
failed_requests: current.failed_requests + (metrics.failed_requests || 0),
tokens: current.tokens + metrics.total_tokens,
cache_read_input_tokens: (current.cache_read_input_tokens ?? 0) + (metrics.cache_read_input_tokens || 0),
cache_creation_input_tokens:
(current.cache_creation_input_tokens ?? 0) + (metrics.cache_creation_input_tokens || 0),
},
};
}, {});
const totalRequests = Object.values(modelBreakdown).reduce((total, model) => total + model.requests, 0);
modelMetrics[metricName].top_models = Object.values(modelBreakdown)
.map((model) => ({
...model,
request_share: totalRequests === 0 ? 0 : (model.requests / totalRequests) * 100,
}))
.sort((a, b) => b.requests - a.requests);
});
}

4
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -27517,6 +27517,10 @@ export interface components {
[key: string]: unknown;
};
metrics: components["schemas"]["SpendMetrics"];
/** Model Breakdown */
model_breakdown?: {
[key: string]: components["schemas"]["SpendMetrics"];
};
};
/** Mode */
Mode: {