From 7b65185a68e55f5efd1b78b866f03211144b880f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 06:23:06 +0000 Subject: [PATCH] fix(ui): stop daily_with_models export from overcounting keys shared across entities --- ui/litellm-dashboard/eslint-metrics.json | 4 +- .../EntityUsageExport/utils.test.ts | 134 ++++++++++++++++++ .../src/components/EntityUsageExport/utils.ts | 110 +++++++------- 3 files changed, 197 insertions(+), 51 deletions(-) diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json index ded6ab97e1e..6e9ec323061 100644 --- a/ui/litellm-dashboard/eslint-metrics.json +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -1,7 +1,7 @@ { - "@typescript-eslint/no-explicit-any": 1982, + "@typescript-eslint/no-explicit-any": 1979, "complexity": 128, - "local/no-large-inline-object-arg": 512, + "local/no-large-inline-object-arg": 519, "local/no-long-condition-chain": 233, "max-depth": 59, "no-console": 15 diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts index dfeecdc4fa6..fa2000cfbd9 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts @@ -1205,6 +1205,10 @@ describe("EntityUsageExport utils", () => { successful_requests: 10, failed_requests: 0, total_tokens: 700, + prompt_tokens: 500, + completion_tokens: 200, + cache_read_input_tokens: 300, + cache_creation_input_tokens: 120, }, metadata: { team_id: "team-1" }, }, @@ -1215,6 +1219,10 @@ describe("EntityUsageExport utils", () => { successful_requests: 15, failed_requests: 0, total_tokens: 350, + prompt_tokens: 250, + completion_tokens: 100, + cache_read_input_tokens: 150, + cache_creation_input_tokens: 80, }, metadata: { team_id: "team-1" }, }, @@ -1645,6 +1653,132 @@ describe("EntityUsageExport utils", () => { expect(result).toHaveLength(0); }); + + it("should not overcount when a key is shared across entities (regression for #32581)", () => { + const data: EntitySpendData = { + results: [ + { + date: "2025-04-01", + breakdown: { + entities: { + tagA: { + metrics: { + spend: 10.0, + api_requests: 100, + successful_requests: 100, + failed_requests: 0, + total_tokens: 1000, + prompt_tokens: 600, + completion_tokens: 400, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + api_key_breakdown: { + key1: { + metrics: { + spend: 10.0, + api_requests: 100, + successful_requests: 100, + failed_requests: 0, + total_tokens: 1000, + prompt_tokens: 600, + completion_tokens: 400, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + metadata: { team_id: "team-1" }, + }, + }, + }, + tagB: { + metrics: { + spend: 10.0, + api_requests: 100, + successful_requests: 100, + failed_requests: 0, + total_tokens: 1000, + prompt_tokens: 600, + completion_tokens: 400, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + api_key_breakdown: { + key1: { + metrics: { + spend: 10.0, + api_requests: 100, + successful_requests: 100, + failed_requests: 0, + total_tokens: 1000, + prompt_tokens: 600, + completion_tokens: 400, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + metadata: { team_id: "team-1" }, + }, + }, + }, + }, + models: { + "gpt-4": { + metrics: { + spend: 20.0, + api_requests: 200, + successful_requests: 200, + failed_requests: 0, + total_tokens: 2000, + }, + api_key_breakdown: { + key1: { + metrics: { + spend: 20.0, + api_requests: 200, + successful_requests: 200, + failed_requests: 0, + total_tokens: 2000, + prompt_tokens: 1200, + completion_tokens: 800, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + metadata: { team_id: "team-1" }, + }, + }, + }, + }, + }, + }, + ], + metadata: { + total_spend: 20.0, + total_api_requests: 200, + total_successful_requests: 200, + total_failed_requests: 0, + total_tokens: 2000, + }, + }; + + const result = generateDailyWithModelsData(data, "Tag"); + + const tagA = result.find((r) => r["Tag ID"] === "tagA"); + const tagB = result.find((r) => r["Tag ID"] === "tagB"); + + expect(tagA?.["Spend ($)"]).toBe("10.0000"); + expect(tagA?.["Total Tokens"]).toBe(1000); + expect(tagA?.["Prompt Tokens"]).toBe(600); + expect(tagA?.Requests).toBe(100); + expect(tagB?.["Spend ($)"]).toBe("10.0000"); + expect(tagB?.["Total Tokens"]).toBe(1000); + + const totalSpend = result.reduce((sum, r) => sum + parseFloat(r["Spend ($)"].replace(/,/g, "")), 0); + const totalTokens = result.reduce((sum, r) => sum + r["Total Tokens"], 0); + const totalRequests = result.reduce((sum, r) => sum + r.Requests, 0); + + expect(totalSpend).toBeCloseTo(20.0, 4); + expect(totalTokens).toBe(2000); + expect(totalRequests).toBe(200); + }); }); describe("generateExportData", () => { diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts index 8ff389ded5e..3b09af6e5ad 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.ts @@ -232,6 +232,51 @@ export const generateDailyWithKeysData = ( return dailyKeyBreakdown.sort((a, b) => new Date(a.Date).getTime() - new Date(b.Date).getTime()); }; +type MetricTotals = Record; +type KeyEntry = { metrics?: MetricTotals }; +type ModelEntry = { api_key_breakdown?: Record }; + +const emptyMetricTotals = (): MetricTotals => Object.fromEntries(METRIC_KEYS.map((metric) => [metric, 0])); + +const sumKeyTotalsAcrossModels = (models: Record): Record => { + const keyTotals: Record = {}; + Object.values(models).forEach((modelData) => { + Object.entries(modelData.api_key_breakdown || {}).forEach(([apiKey, keyData]) => { + const totals = keyTotals[apiKey] || (keyTotals[apiKey] = emptyMetricTotals()); + const metrics = keyData.metrics || {}; + METRIC_KEYS.forEach((metric) => { + totals[metric] += metrics[metric] || 0; + }); + }); + }); + return keyTotals; +}; + +const aggregateEntityModelMetrics = ( + entityApiKeys: Record, + models: Record, + keyTotalsAcrossModels: Record, +): Record => { + const entityModels: Record = {}; + Object.entries(entityApiKeys).forEach(([apiKey, entityKeyData]) => { + const keyTotals = keyTotalsAcrossModels[apiKey]; + if (!keyTotals) return; + const entityKeyMetrics = entityKeyData.metrics || {}; + + Object.entries(models).forEach(([model, modelData]) => { + const modelKeyMetrics = modelData.api_key_breakdown?.[apiKey]?.metrics; + if (!modelKeyMetrics) return; + const bucket = entityModels[model] || (entityModels[model] = emptyMetricTotals()); + METRIC_KEYS.forEach((metric) => { + const keyTotal = keyTotals[metric]; + const entityShare = keyTotal > 0 ? (entityKeyMetrics[metric] || 0) / keyTotal : 0; + bucket[metric] += (modelKeyMetrics[metric] || 0) * entityShare; + }); + }); + }); + return entityModels; +}; + export const generateDailyWithModelsData = ( spendData: EntitySpendData, entityLabel: string, @@ -240,65 +285,32 @@ export const generateDailyWithModelsData = ( const dailyModelBreakdown: any[] = []; spendData.results.forEach((day) => { - const dailyEntityModels: { [key: string]: { [key: string]: any } } = {}; + const models = day.breakdown.models || {}; + const keyTotalsAcrossModels = sumKeyTotalsAcrossModels(models); Object.entries(resolveEntities(day.breakdown)).forEach(([entity, entityData]: [string, any]) => { - if (!dailyEntityModels[entity]) { - dailyEntityModels[entity] = {}; - } - - Object.entries(day.breakdown.models || {}).forEach(([model, modelData]: [string, any]) => { - const entityApiKeys = entityData.api_key_breakdown || {}; - const modelApiKeys = modelData.api_key_breakdown || {}; - - Object.keys(entityApiKeys).forEach((apiKey) => { - const keyMetrics = modelApiKeys[apiKey]?.metrics; - if (!keyMetrics) return; - - if (!dailyEntityModels[entity][model]) { - dailyEntityModels[entity][model] = { - spend: 0, - requests: 0, - successful: 0, - failed: 0, - tokens: 0, - promptTokens: 0, - completionTokens: 0, - cacheReadInputTokens: 0, - cacheCreationInputTokens: 0, - }; - } - dailyEntityModels[entity][model].spend += keyMetrics.spend || 0; - dailyEntityModels[entity][model].requests += keyMetrics.api_requests || 0; - dailyEntityModels[entity][model].successful += keyMetrics.successful_requests || 0; - dailyEntityModels[entity][model].failed += keyMetrics.failed_requests || 0; - dailyEntityModels[entity][model].tokens += keyMetrics.total_tokens || 0; - dailyEntityModels[entity][model].promptTokens += keyMetrics.prompt_tokens || 0; - dailyEntityModels[entity][model].completionTokens += keyMetrics.completion_tokens || 0; - dailyEntityModels[entity][model].cacheReadInputTokens += keyMetrics.cache_read_input_tokens || 0; - dailyEntityModels[entity][model].cacheCreationInputTokens += keyMetrics.cache_creation_input_tokens || 0; - }); - }); - }); - - Object.entries(dailyEntityModels).forEach(([entity, models]) => { const { id, alias } = resolveEntityDisplay(entity, teamAliasMap); + const entityModels = aggregateEntityModelMetrics( + entityData.api_key_breakdown || {}, + models, + keyTotalsAcrossModels, + ); - Object.entries(models).forEach(([model, metrics]: [string, any]) => { + Object.entries(entityModels).forEach(([model, metrics]) => { dailyModelBreakdown.push({ Date: day.date, [entityLabel]: alias, [`${entityLabel} ID`]: id, Model: model, "Spend ($)": formatNumberWithCommas(metrics.spend, 4), - Requests: metrics.requests, - Successful: metrics.successful, - Failed: metrics.failed, - "Total Tokens": metrics.tokens, - "Prompt Tokens": metrics.promptTokens, - "Completion Tokens": metrics.completionTokens, - "Cache Read Input Tokens": metrics.cacheReadInputTokens, - "Cache Creation Input Tokens": metrics.cacheCreationInputTokens, + Requests: metrics.api_requests, + Successful: metrics.successful_requests, + Failed: metrics.failed_requests, + "Total Tokens": metrics.total_tokens, + "Prompt Tokens": metrics.prompt_tokens, + "Completion Tokens": metrics.completion_tokens, + "Cache Read Input Tokens": metrics.cache_read_input_tokens, + "Cache Creation Input Tokens": metrics.cache_creation_input_tokens, }); }); });