From 2701e2008baee4ed2c16cba7a56ea7f96d6d0981 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 23:51:51 -0700 Subject: [PATCH] fix(ui): group cost optimization cache leakage by model group (#43008) * test(ui): cover cost optimization cache leakage grouping by model group Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): group cost optimization cache leakage by model group Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/ui/fixtures/pages.ts | 1 + .../costOptimizationModelGroups.spec.ts | 51 ++++++++++++++++++ .../tests/integrationCritical/expected.json | 3 +- .../integration/_support/daily_spend_rows.py | 52 +++++++++++++++++++ tests/integration/_support/database.py | 7 ++- .../_components/CacheLeakageCard.test.tsx | 4 +- .../_components/costOptimizationUtils.test.ts | 52 ++++++++++++++++++- .../_components/costOptimizationUtils.ts | 2 +- 8 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 tests/e2e/ui/tests/integrationCritical/costOptimizationModelGroups.spec.ts create mode 100644 tests/integration/_support/daily_spend_rows.py diff --git a/tests/e2e/ui/fixtures/pages.ts b/tests/e2e/ui/fixtures/pages.ts index 56b2bed380d..ba5887f3113 100644 --- a/tests/e2e/ui/fixtures/pages.ts +++ b/tests/e2e/ui/fixtures/pages.ts @@ -20,6 +20,7 @@ export enum Page { RouterSettings = "router-settings", UiTheme = "ui-theme", CostTracking = "cost-tracking", + CostOptimization = "cost-optimization", ModelHubTable = "model-hub-table", Caching = "caching", Logs = "logs", diff --git a/tests/e2e/ui/tests/integrationCritical/costOptimizationModelGroups.spec.ts b/tests/e2e/ui/tests/integrationCritical/costOptimizationModelGroups.spec.ts new file mode 100644 index 00000000000..08dcc4c2c21 --- /dev/null +++ b/tests/e2e/ui/tests/integrationCritical/costOptimizationModelGroups.spec.ts @@ -0,0 +1,51 @@ +import { test, expect } from "@playwright/test"; +import { randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import * as path from "node:path"; +import { Page } from "../../fixtures/pages"; +import { navigateToPage } from "../../helpers/navigation"; + +test("cache leakage by model merges a deployment's resolved and requested model names into its model group", async ({ + page, +}) => { + const master = process.env.LITELLM_MASTER_KEY ?? "sk-integration-master"; + const marker = `integration-browser-${randomUUID()}`; + const group = `${marker}-public`; + const deployment = `${marker}-backend`; + const apiKey = marker; + const support = (...args: string[]) => + execFileSync( + process.env.INTEGRATION_PYTHON ?? "python", + [ + path.resolve( + __dirname, + "../../../../integration/_support/daily_spend_rows.py", + ), + ...args, + ], + { encoding: "utf8", timeout: 10_000, killSignal: "SIGKILL" }, + ); + try { + support("seed", apiKey, group, deployment); + await page.goto("/ui/login"); + await page.getByPlaceholder("Enter your username").fill("admin"); + await page.getByPlaceholder("Enter your password").fill(master); + await page.getByRole("button", { name: "Login", exact: true }).click(); + await expect(page).toHaveURL( + (url) => + url.pathname.startsWith("/ui") && !url.pathname.includes("login"), + ); + await navigateToPage(page, Page.CostOptimization); + await page.getByRole("tab", { name: "Prompt Caching" }).click(); + await page.getByRole("tab", { name: "By model" }).click(); + const rows = page.getByRole("row").filter({ hasText: marker }); + await expect(rows).toHaveCount(1); + await expect(rows.first()).toContainText(group); + await expect(rows.first()).toContainText("275,000"); + await expect( + page.getByRole("row").filter({ hasText: deployment }), + ).toHaveCount(0); + } finally { + support("clear", apiKey); + } +}); diff --git a/tests/e2e/ui/tests/integrationCritical/expected.json b/tests/e2e/ui/tests/integrationCritical/expected.json index 189cdef9a93..b73c04acbe0 100644 --- a/tests/e2e/ui/tests/integrationCritical/expected.json +++ b/tests/e2e/ui/tests/integrationCritical/expected.json @@ -6,5 +6,6 @@ "tests/e2e/ui/tests/integrationCritical/mcpUserEnvVars.spec.ts::pressing Enter on Update opens the credentials modal instead of the server editor", "tests/e2e/ui/tests/integrationCritical/mcpUserEnvVars.spec.ts::a server with two per-user variables reports the remaining gap until both are saved", "tests/e2e/ui/tests/integrationCritical/mcpUserEnvVars.spec.ts::a server without per-user variables shows no credential row", - "tests/e2e/ui/tests/integrationCritical/mcpUserEnvVars.spec.ts::clearing credentials for a server deleted underneath the modal reports the failure without losing the page" + "tests/e2e/ui/tests/integrationCritical/mcpUserEnvVars.spec.ts::clearing credentials for a server deleted underneath the modal reports the failure without losing the page", + "tests/e2e/ui/tests/integrationCritical/costOptimizationModelGroups.spec.ts::cache leakage by model merges a deployment's resolved and requested model names into its model group" ] diff --git a/tests/integration/_support/daily_spend_rows.py b/tests/integration/_support/daily_spend_rows.py new file mode 100644 index 00000000000..9d530fb436a --- /dev/null +++ b/tests/integration/_support/daily_spend_rows.py @@ -0,0 +1,52 @@ +import json +import sys +from datetime import datetime, timezone +from typing import Final, LiteralString + +from integration._support.database import read_rows, write_rows + +SEED_QUERY: Final[LiteralString] = """ +INSERT INTO "LiteLLM_DailyUserSpend" ( + id, user_id, date, api_key, model, model_group, custom_llm_provider, + endpoint, mcp_namespaced_tool_name, + prompt_tokens, completion_tokens, spend, + api_requests, successful_requests, failed_requests, updated_at +) VALUES ( + gen_random_uuid()::text, %s, %s, %s, %s, %s, 'bedrock', + NULL, NULL, + %s, %s, %s, + %s, %s, %s, now() +) +""" + +CLEAR_COUNT_QUERY: Final[LiteralString] = 'SELECT count(*) AS count FROM "LiteLLM_DailyUserSpend" WHERE api_key=%s' +CLEAR_QUERY: Final[LiteralString] = 'DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=%s' + + +def seed(api_key: str, group: str, deployment: str) -> int: + today: Final = datetime.now(timezone.utc).date().isoformat() + write_rows( + SEED_QUERY, + (api_key, today, api_key, deployment, group, "270000", "1000", "0.81", "270", "270", "0"), + ) + write_rows( + SEED_QUERY, + (api_key, today, api_key, group, "", "5000", "0", "0.0", "50", "0", "50"), + ) + return 2 + + +def clear(api_key: str) -> int: + count: Final = int(str(read_rows(CLEAR_COUNT_QUERY, (api_key,))[0]["count"])) + write_rows(CLEAR_QUERY, (api_key,)) + return count + + +if __name__ == "__main__": + command: Final = sys.argv[1] + if command == "seed": + sys.stdout.write(json.dumps({"affected": seed(sys.argv[2], sys.argv[3], sys.argv[4])}) + "\n") + elif command == "clear": + sys.stdout.write(json.dumps({"affected": clear(sys.argv[2])}) + "\n") + else: + raise SystemExit(f"unknown command: {command}") diff --git a/tests/integration/_support/database.py b/tests/integration/_support/database.py index e7f0ebdf603..461cdbda1ee 100644 --- a/tests/integration/_support/database.py +++ b/tests/integration/_support/database.py @@ -1,5 +1,5 @@ import os -from typing import Final +from typing import Final, LiteralString import psycopg from psycopg.rows import dict_row @@ -14,3 +14,8 @@ def read_rows( with psycopg.connect(database_url or os.environ["DATABASE_URL"], row_factory=dict_row) as connection: connection.execute("SET TRANSACTION READ ONLY") return ROWS.validate_python(connection.execute(query, parameters).fetchall()) + + +def write_rows(query: LiteralString, parameters: tuple[str, ...], *, database_url: str | None = None) -> None: + with psycopg.connect(database_url or os.environ["DATABASE_URL"]) as connection: + connection.execute(query, parameters) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx index 1c39c6eb36d..b06986d01f0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CacheLeakageCard.test.tsx @@ -46,13 +46,13 @@ const dayWithModels = (date: string, models: Record [ name, { metrics: baseMetrics(m), metadata: {}, api_key_breakdown: {} }, ]), ), - model_groups: {}, mcp_servers: {}, providers: {}, api_keys: {}, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts index 9c3915c812f..20b64179ce8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.test.ts @@ -56,10 +56,10 @@ const modelDay = (date: string, models: Record>): date, metrics: metrics({}), breakdown: { - models: Object.fromEntries( + models: {}, + model_groups: Object.fromEntries( Object.entries(models).map(([name, m]) => [name, { metrics: metrics(m), metadata: {}, api_key_breakdown: {} }]), ), - model_groups: {}, mcp_servers: {}, providers: {}, entities: {}, @@ -244,6 +244,54 @@ describe("computeCacheLeakage by model", () => { expect(rows.map((r) => r.id)).toEqual(["gemini-2.5-flash"]); expect(rows[0].potentialSavings).toBeCloseTo(1.0, 6); }); + + it("merges rows logged under a deployment's resolved and requested names into one model group row", () => { + const day: DailyData = { + date: "2026-07-01", + metrics: metrics({}), + breakdown: { + models: { + "bedrock/global.anthropic.claude-sonnet-4-6": { + metrics: metrics({ prompt_tokens: 270000 }), + metadata: {}, + api_key_breakdown: {}, + }, + "bedrock/claude-sonnet-4-6": { + metrics: metrics({ prompt_tokens: 5000 }), + metadata: {}, + api_key_breakdown: {}, + }, + }, + model_groups: { + "bedrock/claude-sonnet-4-6": { + metrics: metrics({ prompt_tokens: 275000 }), + metadata: {}, + api_key_breakdown: {}, + }, + }, + mcp_servers: {}, + providers: {}, + entities: {}, + api_keys: {}, + }, + }; + const { rows } = computeCacheLeakage([day], "model"); + expect(rows.map((r) => r.id)).toEqual(["bedrock/claude-sonnet-4-6"]); + expect(rows[0].uncachedPromptTokens).toBe(275000); + }); + + it("sums a model group across days", () => { + const results = [ + modelDay("2026-07-01", { "bedrock/claude-sonnet-4-6": { prompt_tokens: 1000 } }), + modelDay("2026-07-02", { + "bedrock/claude-sonnet-4-6": { prompt_tokens: 2500, cache_read_input_tokens: 500 }, + }), + ]; + const { rows } = computeCacheLeakage(results, "model"); + expect(rows).toHaveLength(1); + expect(rows[0].uncachedPromptTokens).toBe(3000); + expect(rows[0].cacheHitRatio).toBeCloseTo(500 / 3500, 6); + }); }); describe("buildDailyToolSeries", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts index 2e6d8208989..5f16b1b04fd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/costOptimizationUtils.ts @@ -93,7 +93,7 @@ const aggregateByKey = (results: readonly DailyData[]): Map => { const byModel = new Map(); for (const day of results) { - for (const [model, entry] of Object.entries(day.breakdown?.models ?? {})) { + for (const [model, entry] of Object.entries(day.breakdown?.model_groups ?? {})) { const acc = byModel.get(model) ?? emptyAccumulator(); byModel.set(model, addMetrics(acc, entry.metrics, null, null)); }