diff --git a/db_scripts/backfill_key_total_spend.sql b/db_scripts/backfill_key_total_spend.sql new file mode 100644 index 00000000000..634a2ac70ce --- /dev/null +++ b/db_scripts/backfill_key_total_spend.sql @@ -0,0 +1,43 @@ +-- One-shot backfill of LiteLLM_VerificationToken.total_spend (lifetime spend) +-- for keys created before the column was introduced in LiteLLM v1.103.0. +-- +-- The column was added with DEFAULT 0 and no backfill, so keys that predate +-- the upgrade report lifetime spend below their current period spend. New +-- deployments do not need this script: total_spend is updated at request +-- time from the moment the release is deployed. Run it only if you want +-- pre-upgrade keys to show their historical lifetime spend. It sets lifetime +-- spend to at least the current spend on every key, active and archived, +-- because current period spend is a valid lower bound on lifetime spend. +-- For keys with no budget reset that is already the exact lifetime value; +-- for resetting keys it only recovers the current period. It is idempotent: +-- it only touches rows where total_spend is below spend, so re-running is a +-- no-op. It touches no spend logs and runs in seconds. +-- +-- IMPORTANT caveats before running: +-- +-- 1. Take a backup of the affected tables first: +-- pg_dump "$DATABASE_URL" -t '"LiteLLM_VerificationToken"' -t '"LiteLLM_DeletedVerificationToken"' > key_total_spend_backup.sql +-- +-- 2. A key "resets" when its own budget_duration IS NOT NULL, or when its +-- budget_id links to a LiteLLM_BudgetTable row whose budget_duration IS +-- NOT NULL (a linked budget resets the key's spend each period too). For +-- those keys this script only recovers the current period; +-- db_scripts/backfill_key_total_spend_from_spend_logs.sql is an optional +-- follow-up that rebuilds the earlier periods from LiteLLM_SpendLogs. +-- +-- 3. No proxy restart is needed. The proxy picks up the corrected values on +-- its next read of each key. +-- +-- Usage: +-- psql "$DATABASE_URL" -f db_scripts/backfill_key_total_spend.sql + +UPDATE "LiteLLM_VerificationToken" +SET total_spend = spend +WHERE total_spend < spend; + +UPDATE "LiteLLM_DeletedVerificationToken" +SET total_spend = spend +WHERE total_spend < spend; + +-- Verify: this should return 0. +-- SELECT count(*) FROM "LiteLLM_VerificationToken" WHERE total_spend < spend; diff --git a/db_scripts/backfill_key_total_spend_from_spend_logs.sql b/db_scripts/backfill_key_total_spend_from_spend_logs.sql new file mode 100644 index 00000000000..b725437e359 --- /dev/null +++ b/db_scripts/backfill_key_total_spend_from_spend_logs.sql @@ -0,0 +1,89 @@ +-- Optional follow-up to db_scripts/backfill_key_total_spend.sql. Run that +-- script first; this one rebuilds earlier budget periods for the keys it +-- can only partially fix: keys whose spend resets each period, because their own +-- budget_duration IS NOT NULL or because their budget_id links to a +-- LiteLLM_BudgetTable row whose budget_duration IS NOT NULL. +-- +-- For those keys the "spend" column only covers the current period, so +-- lifetime spend is reconstructed from LiteLLM_SpendLogs. The join matches +-- l.api_key against both the stored token and its second sha256 +-- (encode(sha256(convert_to(token, 'UTF8')), 'hex')), because spend logs +-- written by older paths recorded the re-hashed digest instead of the +-- token. It is idempotent and never lowers a value: every statement only +-- touches rows where total_spend is below the rebuilt sum, so re-running is +-- a no-op, and a key whose log history is shorter than its current period +-- keeps the value backfill_key_total_spend.sql already gave it. +-- +-- IMPORTANT caveats before running: +-- +-- 1. Take a backup of the affected tables first: +-- pg_dump "$DATABASE_URL" -t '"LiteLLM_VerificationToken"' -t '"LiteLLM_DeletedVerificationToken"' > key_total_spend_backup.sql +-- +-- 2. It requires spend logs to have been enabled, and coverage is bounded +-- by maximum_spend_logs_retention_period: spend older than the retention +-- window is already gone and cannot be recovered. +-- +-- 3. On a large SpendLogs table the join scan is slow, so run it off peak. +-- +-- 4. Run it while the proxy is idle (or with traffic paused). The proxy +-- flushes spend logs in batches, so a request that already raised +-- total_spend but whose log is still queued is missing from the sum, and +-- the rebuilt value would be short by that in-flight amount. +-- +-- 5. A custom token can be deleted and recreated, so the archived table can +-- hold several lifetimes of one token. The update only rewrites archived +-- rows that reset, and the log sum covers every lifetime of that token. +-- +-- 6. No proxy restart is needed. The proxy picks up the corrected values on +-- its next read of each key. +-- +-- Usage: +-- psql "$DATABASE_URL" -f db_scripts/backfill_key_total_spend_from_spend_logs.sql + +-- Active keys whose spend resets (own budget_duration, or a linked +-- LiteLLM_BudgetTable row with one). Rebuild from LiteLLM_SpendLogs, +-- matching api_key against the stored token and its second sha256 digest. +UPDATE "LiteLLM_VerificationToken" k +SET total_spend = s.sum_spend +FROM ( + SELECT k2.token, SUM(l.spend) AS sum_spend + FROM "LiteLLM_VerificationToken" k2 + JOIN "LiteLLM_SpendLogs" l + ON l.api_key IN (k2.token, encode(sha256(convert_to(k2.token, 'UTF8')), 'hex')) + WHERE k2.budget_duration IS NOT NULL + OR k2.budget_id IN ( + SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL + ) + GROUP BY k2.token +) s +WHERE k.token = s.token + AND k.total_spend < s.sum_spend; + +-- Archived tokens are not unique, so collapse them to one row per token +-- before joining spend logs; the update then hits every resetting archived +-- row. +UPDATE "LiteLLM_DeletedVerificationToken" k +SET total_spend = s.sum_spend +FROM ( + SELECT k2.token, SUM(l.spend) AS sum_spend + FROM ( + SELECT DISTINCT token + FROM "LiteLLM_DeletedVerificationToken" + WHERE budget_duration IS NOT NULL + OR budget_id IN ( + SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL + ) + ) k2 + JOIN "LiteLLM_SpendLogs" l + ON l.api_key IN (k2.token, encode(sha256(convert_to(k2.token, 'UTF8')), 'hex')) + GROUP BY k2.token +) s +WHERE k.token = s.token + AND k.total_spend < s.sum_spend + AND (k.budget_duration IS NOT NULL + OR k.budget_id IN ( + SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL + )); + +-- Verify: this should return 0. +-- SELECT count(*) FROM "LiteLLM_VerificationToken" WHERE total_spend < spend; diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx index 33625c0e1eb..477d7b0ecb4 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx @@ -295,7 +295,7 @@ export const getKeyTableColumns = ({ header: () => ( ), size: 130, diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx index 4bf41c1f3a8..48f2f47ceda 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx @@ -7,7 +7,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { KeyResponse, Team } from "../key_team_helpers/key_list"; import { keyDeleteCall, keyUpdateCall } from "../networking"; import { QueryClient } from "@tanstack/react-query"; -import KeyInfoView from "./key_info_view"; +import KeyInfoView, { needsLifetimeSpendBackfill } from "./key_info_view"; const editViewMocks = vi.hoisted(() => ({ onSubmit: undefined as ((v: Record) => Promise) | undefined, @@ -290,6 +290,58 @@ describe("KeyInfoView", () => { expect(screen.getByTestId("key-lifetime-spend")).toHaveTextContent("Lifetime spend: $340.5000"); }); + it("shows the backfill hint when lifetime spend trails the period spend", async () => { + vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); + + renderWithProviders( + {}} + keyId={"test-key-id"} + onKeyDataUpdate={() => {}} + teams={[]} + />, + ); + + expect(await screen.findByTestId("key-lifetime-spend-backfill-hint")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /lifetime spend is below/i })).toBeInTheDocument(); + expect(screen.getByTestId("key-lifetime-spend")).toHaveTextContent("Lifetime spend: $4.0000"); + }); + + it("hides the backfill hint when lifetime spend covers the period spend", async () => { + vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); + + renderWithProviders( + {}} + keyId={"test-key-id"} + onKeyDataUpdate={() => {}} + teams={[]} + />, + ); + + expect(await screen.findByTestId("key-lifetime-spend")).toBeInTheDocument(); + expect(screen.queryByTestId("key-lifetime-spend-backfill-hint")).not.toBeInTheDocument(); + }); + + describe("needsLifetimeSpendBackfill", () => { + it("returns true when total spend is below the period spend", () => { + expect(needsLifetimeSpendBackfill(10, 4)).toBe(true); + }); + + it("returns false when total spend equals or exceeds the period spend", () => { + expect(needsLifetimeSpendBackfill(10, 10)).toBe(false); + expect(needsLifetimeSpendBackfill(10, 12)).toBe(false); + }); + + it("treats a missing total spend as zero", () => { + expect(needsLifetimeSpendBackfill(10, null)).toBe(true); + expect(needsLifetimeSpendBackfill(10, undefined)).toBe(true); + expect(needsLifetimeSpendBackfill(0, null)).toBe(false); + }); + }); + it("should render the key's saved router fallbacks", async () => { vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock); diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 3c2c9bb352d..63693fd1af5 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -6,11 +6,12 @@ import useTeams from "@/app/(dashboard)/hooks/useTeams"; import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; import { formatNumberWithCommas } from "@/utils/dataUtils"; import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils"; -import { ArrowLeft } from "lucide-react"; +import { ArrowLeft, Info } from "lucide-react"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EntityLink } from "@/components/shared/EntityLink"; import { modelGroupHref, teamDetailHref } from "@/utils/entityLinks"; @@ -49,6 +50,10 @@ import { parseErrorMessage } from "../shared/errorUtils"; import { InheritedBudgetHint, inheritedBudgetGates, keyOwnerBudgetSource } from "../shared/InheritedBudgetHint"; import { KeyEditView } from "./key_edit_view"; +export function needsLifetimeSpendBackfill(spend: number, totalSpend: number | null | undefined): boolean { + return (totalSpend ?? 0) < spend; +} + interface KeyInfoViewProps { keyId: string; onClose: () => void; @@ -682,6 +687,26 @@ export default function KeyInfoView({ )}

Lifetime spend: ${formatNumberWithCommas(currentKeyData.total_spend ?? 0, 4)} + {needsLifetimeSpendBackfill(currentKeyData.spend, currentKeyData.total_spend) && ( + + + } + > + + + + Lifetime tracking started with LiteLLM v1.103.0 on September 19, 2026 and was not backfilled, + so this key's lifetime spend only counts usage since that upgrade. + + + )}