scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

fix(usage): nest router models under keys

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Krrish Dholakia 2026-07-16 07:45:27 +00:00
parent ac827e669a
commit 3fb0044e4b
8 changed files with 283 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
litellm/proxy/management_endpoints/common_daily_activity.py
View file

@ -154,6 +154,14 @@ def update_breakdown_metrics(
breakdown.model_groups[record.model_group].api_key_breakdown[record.api_key].metrics,
record,
)
if record.model:
key_metrics = breakdown.model_groups[record.model_group].api_key_breakdown[record.api_key]
if record.model not in key_metrics.model_breakdown:
key_metrics.model_breakdown[record.model] = SpendMetrics()
key_metrics.model_breakdown[record.model] = update_metrics(
key_metrics.model_breakdown[record.model],
record,
)
if record.mcp_namespaced_tool_name:
if record.mcp_namespaced_tool_name not in breakdown.mcp_servers:
@ -493,6 +501,7 @@ def _build_aggregated_sql_query(
(date, model_group),
(date, model, model_group),
(date, model_group, api_key),
(date, api_key, model, model_group),
(date, custom_llm_provider),
(date, custom_llm_provider, api_key),
(date, mcp_namespaced_tool_name),
@ -598,6 +607,7 @@ _GROUP_DATE_MODEL_API_KEY = 15 # 0b0001111
_GROUP_DATE_MODEL_GROUP = 55 # 0b0110111
_GROUP_DATE_MODEL_MODEL_GROUP = 39 # 0b0100111
_GROUP_DATE_MODEL_GROUP_API_KEY = 23 # 0b0010111
_GROUP_DATE_MODEL_MODEL_GROUP_API_KEY = 7 # 0b0000111
_GROUP_DATE_PROVIDER = 59 # 0b0111011
_GROUP_DATE_PROVIDER_API_KEY = 27 # 0b0011011
_GROUP_DATE_MCP = 61 # 0b0111101
@ -671,9 +681,14 @@ def _aggregate_grouping_sets_records_sync(
if parent is None:
parent = MetricWithMetadata(metrics=SpendMetrics(), metadata={})
target[parent_key] = parent
parent.api_key_breakdown[api_key] = KeyMetricWithMetadata(
metrics=metrics, metadata=_key_metadata(api_key_metadata, api_key)
)
existing = parent.api_key_breakdown.get(api_key)
if existing is None:
parent.api_key_breakdown[api_key] = KeyMetricWithMetadata(
metrics=metrics, metadata=_key_metadata(api_key_metadata, api_key)
)
else:
existing.metrics = metrics
existing.metadata = _key_metadata(api_key_metadata, api_key)
for record in records:
level = record.group_level
@ -719,6 +734,20 @@ def _aggregate_grouping_sets_records_sync(
record.api_key,
metrics,
)
elif level == _GROUP_DATE_MODEL_MODEL_GROUP_API_KEY:
if record.model_group and record.model and record.api_key:
parent = breakdown.model_groups.get(record.model_group)
if parent is None:
parent = MetricWithMetadata(metrics=SpendMetrics(), metadata={})
breakdown.model_groups[record.model_group] = parent
key_metrics = parent.api_key_breakdown.get(record.api_key)
if key_metrics is None:
key_metrics = KeyMetricWithMetadata(
metrics=SpendMetrics(),
metadata=_key_metadata(api_key_metadata, record.api_key),
)
parent.api_key_breakdown[record.api_key] = key_metrics
key_metrics.model_breakdown[record.model] = metrics
elif level == _GROUP_DATE_PROVIDER:
provider = record.custom_llm_provider or "unknown"
assign_metric_with_metadata(breakdown.providers, provider, metrics)

1
litellm/types/proxy/management_endpoints/common_daily_activity.py
View file

@ -43,6 +43,7 @@ class KeyMetricWithMetadata(MetricBase):
"""Base class for metrics with additional metadata"""
metadata: KeyMetadata = Field(default_factory=KeyMetadata)
model_breakdown: Dict[str, SpendMetrics] = Field(default_factory=dict)
class MetricWithMetadata(MetricBase):

53
tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py
View file

@ -854,6 +854,7 @@ class TestBuildAggregatedSqlQuery:
)
assert "(date, model, model_group)" in sql
assert "(date, api_key, model, model_group)" in sql
def test_aggregate_grouping_sets_populates_model_group_model_breakdown():
@ -887,6 +888,51 @@ def test_aggregate_grouping_sets_populates_model_group_model_breakdown():
assert metrics.cache_creation_input_tokens == 1000
def test_aggregate_grouping_sets_populates_key_model_group_model_breakdown():
model_record = SimpleNamespace(
group_level=7,
date="2026-05-29",
api_key="key-hash",
model="bedrock/claude-opus-4-8",
model_group="smart-router",
custom_llm_provider=None,
mcp_namespaced_tool_name=None,
endpoint=None,
spend=12.5,
prompt_tokens=8000,
completion_tokens=2000,
cache_read_input_tokens=4000,
cache_creation_input_tokens=1000,
api_requests=8,
successful_requests=8,
failed_requests=0,
)
model_group_key_record = SimpleNamespace(
**{
**model_record.__dict__,
"group_level": 23,
"model": None,
}
)
result = _aggregate_grouping_sets_records_sync(
records=[model_record, model_group_key_record],
api_key_metadata={},
)
key_metrics = (
result["results"][0]
.breakdown.model_groups["smart-router"]
.api_key_breakdown["key-hash"]
)
metrics = key_metrics.model_breakdown["bedrock/claude-opus-4-8"]
assert key_metrics.metrics.api_requests == 8
assert metrics.api_requests == 8
assert metrics.total_tokens == 10000
assert metrics.cache_read_input_tokens == 4000
assert metrics.cache_creation_input_tokens == 1000
def test_update_breakdown_metrics_populates_model_group_model_breakdown():
record = SimpleNamespace(
model="bedrock/claude-sonnet-4-5",
@ -920,6 +966,13 @@ def test_update_breakdown_metrics_populates_model_group_model_breakdown():
assert metrics.total_tokens == 4000
assert metrics.cache_read_input_tokens == 1500
assert metrics.cache_creation_input_tokens == 500
key_metrics = result.model_groups["smart-router"].api_key_breakdown["key-hash"].model_breakdown[
"bedrock/claude-sonnet-4-5"
]
assert key_metrics.api_requests == 2
assert key_metrics.total_tokens == 4000
assert key_metrics.cache_read_input_tokens == 1500
assert key_metrics.cache_creation_input_tokens == 500
@pytest.mark.asyncio

5
ui/litellm-dashboard/src/components/UsagePage/components/KeyModelUsageView.tsx
View file

@ -9,6 +9,7 @@ import { TopModelData } from "../types";
interface KeyModelUsageViewProps {
topModels: TopModelData[];
title?: string;
}
const VISIBLE_ROWS = 5;
@ -70,7 +71,7 @@ const columns: ColumnsType<TopModelData> = [
},
];
const KeyModelUsageView: React.FC<KeyModelUsageViewProps> = ({ topModels }) => {
const KeyModelUsageView: React.FC<KeyModelUsageViewProps> = ({ topModels, title = "Model Usage" }) => {
const [viewMode, setViewMode] = useState<"chart" | "table">("table");
if (topModels.length === 0) {
@ -80,7 +81,7 @@ const KeyModelUsageView: React.FC<KeyModelUsageViewProps> = ({ topModels }) => {
return (
<Card className="mt-4">
<CardHeader>
<CardTitle className="text-base font-semibold">Model Usage</CardTitle>
<CardTitle className="text-base font-semibold">{title}</CardTitle>
<CardAction>
<div className="flex space-x-2">
<button

9
ui/litellm-dashboard/src/components/UsagePage/types.ts
View file

@ -36,6 +36,7 @@ export interface MetricWithMetadata {
export interface KeyMetricWithMetadata {
metrics: SpendMetrics;
metadata: KeyMetadata;
model_breakdown?: { [key: string]: SpendMetrics };
}
export interface KeyMetadata {
@ -65,6 +66,13 @@ export interface TopModelData {
cache_creation_input_tokens?: number;
}
export interface ModelGroupUsageData {
model_group: string;
spend: number;
requests: number;
top_models: TopModelData[];
}
export interface ModelActivityData {
label: string;
total_requests: number;
@ -78,6 +86,7 @@ export interface ModelActivityData {
total_spend: number;
top_api_keys: TopApiKeyData[];
top_models: TopModelData[];
top_model_groups?: ModelGroupUsageData[];
daily_data: {
date: string;
metrics: {

97
ui/litellm-dashboard/src/components/activity_metrics.test.tsx
View file

@ -259,6 +259,41 @@ describe("ActivityMetrics", () => {
expect(gpt4Index).toBeLessThan(gpt35Index);
});
it("should display smart router names before underlying model usage", () => {
render(
<ActivityMetrics
modelMetrics={{
"api-key-hash": createMockModelActivityData("key-alias", {
top_model_groups: [
{
model_group: "smart-router",
spend: 10,
requests: 10,
top_models: [
{
model: "bedrock/claude-opus-4-8",
spend: 8,
requests: 8,
request_share: 80,
successful_requests: 8,
failed_requests: 0,
tokens: 8000,
cache_read_input_tokens: 4000,
cache_creation_input_tokens: 1000,
},
],
},
],
}),
}}
/>,
);
expect(screen.getByText("Smart Router Usage")).toBeInTheDocument();
expect(screen.getByText("smart-router")).toBeInTheDocument();
expect(screen.getByText("Underlying Model Usage")).toBeInTheDocument();
});
it("should display model summary cards with correct values", () => {
render(<ActivityMetrics modelMetrics={mockModelMetrics} />);
const requestElements = screen.getAllByText("100");
@ -1174,16 +1209,21 @@ describe("processActivityData", () => {
expect(result).toEqual({});
});
it("should populate top_models for api_keys when models breakdown contains api_key_breakdown for that key", () => {
it("should populate model groups and their underlying models for api keys", () => {
const dailyActivityWithModelsForKey: { results: DailyData[] } = {
results: [
{
date: "2025-01-01",
metrics: EMPTY_SPEND_METRICS,
breakdown: {
models: {
"gpt-4": {
metrics: EMPTY_SPEND_METRICS,
models: {},
model_groups: {
"smart-router": {
metrics: {
...EMPTY_SPEND_METRICS,
spend: 60,
api_requests: 60,
},
metadata: {},
api_key_breakdown: {
"api-key-hash-1": {
@ -1199,11 +1239,32 @@ describe("processActivityData", () => {
cache_creation_input_tokens: 3000,
},
metadata: { key_alias: "key-alias-1", team_id: "team1" },
model_breakdown: {
"bedrock/claude-opus-4-8": {
...EMPTY_SPEND_METRICS,
spend: 48,
api_requests: 48,
total_tokens: 24000,
successful_requests: 46,
failed_requests: 2,
cache_read_input_tokens: 5000,
cache_creation_input_tokens: 2500,
},
"bedrock/claude-sonnet-4-5": {
...EMPTY_SPEND_METRICS,
spend: 12,
api_requests: 12,
total_tokens: 6000,
successful_requests: 11,
failed_requests: 1,
cache_read_input_tokens: 1000,
cache_creation_input_tokens: 500,
},
},
},
},
},
},
model_groups: {},
mcp_servers: {},
providers: {},
api_keys: {
@ -1230,13 +1291,25 @@ describe("processActivityData", () => {
const result = processActivityData(dailyActivityWithModelsForKey, "api_keys", MOCK_TEAMS);
expect(result["api-key-hash-1"].top_models).toHaveLength(1);
expect(result["api-key-hash-1"].top_models[0].model).toBe("gpt-4");
expect(result["api-key-hash-1"].top_models[0].spend).toBe(60.0);
expect(result["api-key-hash-1"].top_models[0].requests).toBe(60);
expect(result["api-key-hash-1"].top_models[0].request_share).toBe(100);
expect(result["api-key-hash-1"].top_models[0].cache_read_input_tokens).toBe(6000);
expect(result["api-key-hash-1"].top_models[0].cache_creation_input_tokens).toBe(3000);
const modelGroup = result["api-key-hash-1"].top_model_groups?.[0];
const expectedOpusMetrics = {
model: "bedrock/claude-opus-4-8",
requests: 48,
request_share: 80,
cache_read_input_tokens: 5000,
cache_creation_input_tokens: 2500,
};
expect(result["api-key-hash-1"].top_models).toEqual([]);
expect(modelGroup?.model_group).toBe("smart-router");
expect(modelGroup?.requests).toBe(60);
expect(modelGroup?.top_models).toEqual([
expect.objectContaining(expectedOpusMetrics),
expect.objectContaining({
model: "bedrock/claude-sonnet-4-5",
requests: 12,
request_share: 20,
}),
]);
});
it("should populate selected model usage for public model groups", () => {

142
ui/litellm-dashboard/src/components/activity_metrics.tsx
View file

@ -9,6 +9,7 @@ import KeyModelUsageView from "./UsagePage/components/KeyModelUsageView";
import {
DailyData,
KeyMetricWithMetadata,
ModelGroupUsageData,
ModelActivityData,
SpendMetrics,
TopApiKeyData,
@ -21,6 +22,30 @@ interface ActivityMetricsProps {
hidePromptCachingMetrics?: boolean;
}
const KeyModelGroupUsageView = ({ modelGroups }: { modelGroups: ModelGroupUsageData[] }) => (
<Card className="mt-4">
<Title>Smart Router Usage</Title>
<Collapse className="mt-3" defaultActiveKey={modelGroups[0]?.model_group}>
{modelGroups.map((modelGroup) => (
<Collapse.Panel
key={modelGroup.model_group}
header={
<div className="flex justify-between items-center w-full">
<Title>{modelGroup.model_group}</Title>
<div className="flex space-x-4 text-sm text-gray-500">
<span>${formatNumberWithCommas(modelGroup.spend, 2)}</span>
<span>{modelGroup.requests.toLocaleString()} requests</span>
</div>
</div>
}
>
<KeyModelUsageView topModels={modelGroup.top_models} title="Underlying Model Usage" />
</Collapse.Panel>
))}
</Collapse>
</Card>
);
const ModelSection = ({
modelName,
metrics,
@ -82,6 +107,10 @@ const ModelSection = ({
{metrics.top_models && metrics.top_models.length > 0 && <KeyModelUsageView topModels={metrics.top_models} />}
{metrics.top_model_groups && metrics.top_model_groups.length > 0 && (
<KeyModelGroupUsageView modelGroups={metrics.top_model_groups} />
)}
{/* Spend per day - Full width card */}
<Card className="mt-4">
<div className="flex justify-between items-center">
@ -395,6 +424,7 @@ export const processActivityData = (
total_cache_creation_input_tokens: 0,
top_api_keys: [],
top_models: [],
top_model_groups: [],
daily_data: [],
};
}
@ -462,57 +492,77 @@ export const processActivityData = (
});
}
if (key === "api_keys" || key === "model_groups") {
Object.entries(modelMetrics).forEach(([metricName]) => {
const modelMetricsByDay = dailyActivity.results.flatMap((day) => {
const perModelMetrics: Record<string, SpendMetrics> =
key === "api_keys"
? Object.fromEntries(
Object.entries(day.breakdown.models || {}).flatMap(([modelName, modelData]) => {
const keyData = modelData.api_key_breakdown?.[metricName];
return keyData ? [[modelName, keyData.metrics]] : [];
}),
)
: day.breakdown.model_groups?.[metricName]?.model_breakdown || {};
const aggregateModelBreakdown = (modelEntries: [string, SpendMetrics][]): TopModelData[] => {
const modelBreakdown = modelEntries.reduce<Record<string, TopModelData>>((totals, [modelName, metrics]) => {
const current = totals[modelName] ?? {
model: modelName,
spend: 0,
requests: 0,
request_share: 0,
successful_requests: 0,
failed_requests: 0,
tokens: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
};
return Object.entries(perModelMetrics);
});
const modelBreakdown = modelMetricsByDay.reduce<Record<string, TopModelData>>((totals, [modelName, metrics]) => {
const current = totals[modelName] ?? {
return {
...totals,
[modelName]: {
model: modelName,
spend: 0,
requests: 0,
spend: current.spend + metrics.spend,
requests: current.requests + metrics.api_requests,
request_share: 0,
successful_requests: 0,
failed_requests: 0,
tokens: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
};
successful_requests: current.successful_requests + (metrics.successful_requests || 0),
failed_requests: current.failed_requests + (metrics.failed_requests || 0),
tokens: current.tokens + metrics.total_tokens,
cache_read_input_tokens: (current.cache_read_input_tokens ?? 0) + (metrics.cache_read_input_tokens || 0),
cache_creation_input_tokens:
(current.cache_creation_input_tokens ?? 0) + (metrics.cache_creation_input_tokens || 0),
},
};
}, {});
const totalRequests = Object.values(modelBreakdown).reduce((total, model) => total + model.requests, 0);
return {
...totals,
[modelName]: {
model: modelName,
spend: current.spend + metrics.spend,
requests: current.requests + metrics.api_requests,
request_share: 0,
successful_requests: current.successful_requests + (metrics.successful_requests || 0),
failed_requests: current.failed_requests + (metrics.failed_requests || 0),
tokens: current.tokens + metrics.total_tokens,
cache_read_input_tokens: (current.cache_read_input_tokens ?? 0) + (metrics.cache_read_input_tokens || 0),
cache_creation_input_tokens:
(current.cache_creation_input_tokens ?? 0) + (metrics.cache_creation_input_tokens || 0),
},
};
}, {});
return Object.values(modelBreakdown)
.map((model) => ({
...model,
request_share: totalRequests === 0 ? 0 : (model.requests / totalRequests) * 100,
}))
.sort((a, b) => b.requests - a.requests);
};
const totalRequests = Object.values(modelBreakdown).reduce((total, model) => total + model.requests, 0);
modelMetrics[metricName].top_models = Object.values(modelBreakdown)
.map((model) => ({
...model,
request_share: totalRequests === 0 ? 0 : (model.requests / totalRequests) * 100,
}))
if (key === "model_groups") {
Object.entries(modelMetrics).forEach(([metricName]) => {
const modelEntries = dailyActivity.results.flatMap((day) =>
Object.entries(day.breakdown.model_groups?.[metricName]?.model_breakdown || {}),
);
modelMetrics[metricName].top_models = aggregateModelBreakdown(modelEntries);
});
}
if (key === "api_keys") {
Object.entries(modelMetrics).forEach(([metricName]) => {
const groupRecords = dailyActivity.results.flatMap((day) =>
Object.entries(day.breakdown.model_groups || {}).flatMap(([modelGroup, groupData]) => {
const keyData = groupData.api_key_breakdown?.[metricName];
return keyData ? [{ modelGroup, keyData }] : [];
}),
);
const modelGroupNames = Array.from(new Set(groupRecords.map(({ modelGroup }) => modelGroup)));
modelMetrics[metricName].top_model_groups = modelGroupNames
.map((modelGroup) => {
const records = groupRecords.filter((record) => record.modelGroup === modelGroup);
return {
model_group: modelGroup,
spend: records.reduce((total, { keyData }) => total + keyData.metrics.spend, 0),
requests: records.reduce((total, { keyData }) => total + keyData.metrics.api_requests, 0),
top_models: aggregateModelBreakdown(
records.flatMap(({ keyData }) => Object.entries(keyData.model_breakdown || {})),
),
};
})
.sort((a, b) => b.requests - a.requests);
});
}

4
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -24599,6 +24599,10 @@ export interface components {
KeyMetricWithMetadata: {
metadata?: components["schemas"]["KeyMetadata"];
metrics: components["schemas"]["SpendMetrics"];
/** Model Breakdown */
model_breakdown?: {
[key: string]: components["schemas"]["SpendMetrics"];
};
};
/** KeyRequest */
KeyRequest: {