fix(ui): explain unbackfilled key lifetime spend and ship a backfill script (#42967)

* fix(ui): explain unbackfilled key lifetime spend and ship a backfill script

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(db_scripts): cover linked budgets, deleted keys and double-hashed logs in total_spend backfill

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(db_scripts): count duplicate archived tokens once in total_spend backfill

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(db_scripts): only rebuild resetting archived rows in total_spend backfill

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(db_scripts): split spend log rebuild into opt-in backfill_key_total_spend_from_spend_logs.sql

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* docs(db_scripts): scope backfill verify query to non-resetting keys

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(db_scripts): lift every key to at least current spend in backfill_key_total_spend.sql

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* docs(db_scripts): align spend log backfill header with lifted floor

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(ui): point resetting keys at the spend log backfill

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(ui): drop script name from lifetime spend tooltip

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: ryan <ryan@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

db_scripts/backfill_key_total_spend.sql

@@ -0,0 +1,43 @@
+-- One-shot backfill of LiteLLM_VerificationToken.total_spend (lifetime spend)
+-- for keys created before the column was introduced in LiteLLM v1.103.0.
+--
+-- The column was added with DEFAULT 0 and no backfill, so keys that predate
+-- the upgrade report lifetime spend below their current period spend. New
+-- deployments do not need this script: total_spend is updated at request
+-- time from the moment the release is deployed. Run it only if you want
+-- pre-upgrade keys to show their historical lifetime spend. It sets lifetime
+-- spend to at least the current spend on every key, active and archived,
+-- because current period spend is a valid lower bound on lifetime spend.
+-- For keys with no budget reset that is already the exact lifetime value;
+-- for resetting keys it only recovers the current period. It is idempotent:
+-- it only touches rows where total_spend is below spend, so re-running is a
+-- no-op. It touches no spend logs and runs in seconds.
+--
+-- IMPORTANT caveats before running:
+--
+-- 1. Take a backup of the affected tables first:
+--      pg_dump "$DATABASE_URL" -t '"LiteLLM_VerificationToken"' -t '"LiteLLM_DeletedVerificationToken"' > key_total_spend_backup.sql
+--
+-- 2. A key "resets" when its own budget_duration IS NOT NULL, or when its
+--    budget_id links to a LiteLLM_BudgetTable row whose budget_duration IS
+--    NOT NULL (a linked budget resets the key's spend each period too). For
+--    those keys this script only recovers the current period;
+--    db_scripts/backfill_key_total_spend_from_spend_logs.sql is an optional
+--    follow-up that rebuilds the earlier periods from LiteLLM_SpendLogs.
+--
+-- 3. No proxy restart is needed. The proxy picks up the corrected values on
+--    its next read of each key.
+--
+-- Usage:
+--   psql "$DATABASE_URL" -f db_scripts/backfill_key_total_spend.sql
+
+UPDATE "LiteLLM_VerificationToken"
+SET total_spend = spend
+WHERE total_spend < spend;
+
+UPDATE "LiteLLM_DeletedVerificationToken"
+SET total_spend = spend
+WHERE total_spend < spend;
+
+-- Verify: this should return 0.
+--   SELECT count(*) FROM "LiteLLM_VerificationToken" WHERE total_spend < spend;

db_scripts/backfill_key_total_spend_from_spend_logs.sql

@@ -0,0 +1,89 @@
+-- Optional follow-up to db_scripts/backfill_key_total_spend.sql. Run that
+-- script first; this one rebuilds earlier budget periods for the keys it
+-- can only partially fix: keys whose spend resets each period, because their own
+-- budget_duration IS NOT NULL or because their budget_id links to a
+-- LiteLLM_BudgetTable row whose budget_duration IS NOT NULL.
+--
+-- For those keys the "spend" column only covers the current period, so
+-- lifetime spend is reconstructed from LiteLLM_SpendLogs. The join matches
+-- l.api_key against both the stored token and its second sha256
+-- (encode(sha256(convert_to(token, 'UTF8')), 'hex')), because spend logs
+-- written by older paths recorded the re-hashed digest instead of the
+-- token. It is idempotent and never lowers a value: every statement only
+-- touches rows where total_spend is below the rebuilt sum, so re-running is
+-- a no-op, and a key whose log history is shorter than its current period
+-- keeps the value backfill_key_total_spend.sql already gave it.
+--
+-- IMPORTANT caveats before running:
+--
+-- 1. Take a backup of the affected tables first:
+--      pg_dump "$DATABASE_URL" -t '"LiteLLM_VerificationToken"' -t '"LiteLLM_DeletedVerificationToken"' > key_total_spend_backup.sql
+--
+-- 2. It requires spend logs to have been enabled, and coverage is bounded
+--    by maximum_spend_logs_retention_period: spend older than the retention
+--    window is already gone and cannot be recovered.
+--
+-- 3. On a large SpendLogs table the join scan is slow, so run it off peak.
+--
+-- 4. Run it while the proxy is idle (or with traffic paused). The proxy
+--    flushes spend logs in batches, so a request that already raised
+--    total_spend but whose log is still queued is missing from the sum, and
+--    the rebuilt value would be short by that in-flight amount.
+--
+-- 5. A custom token can be deleted and recreated, so the archived table can
+--    hold several lifetimes of one token. The update only rewrites archived
+--    rows that reset, and the log sum covers every lifetime of that token.
+--
+-- 6. No proxy restart is needed. The proxy picks up the corrected values on
+--    its next read of each key.
+--
+-- Usage:
+--   psql "$DATABASE_URL" -f db_scripts/backfill_key_total_spend_from_spend_logs.sql
+
+-- Active keys whose spend resets (own budget_duration, or a linked
+-- LiteLLM_BudgetTable row with one). Rebuild from LiteLLM_SpendLogs,
+-- matching api_key against the stored token and its second sha256 digest.
+UPDATE "LiteLLM_VerificationToken" k
+SET total_spend = s.sum_spend
+FROM (
+    SELECT k2.token, SUM(l.spend) AS sum_spend
+    FROM "LiteLLM_VerificationToken" k2
+    JOIN "LiteLLM_SpendLogs" l
+      ON l.api_key IN (k2.token, encode(sha256(convert_to(k2.token, 'UTF8')), 'hex'))
+    WHERE k2.budget_duration IS NOT NULL
+       OR k2.budget_id IN (
+           SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL
+       )
+    GROUP BY k2.token
+) s
+WHERE k.token = s.token
+  AND k.total_spend < s.sum_spend;
+
+-- Archived tokens are not unique, so collapse them to one row per token
+-- before joining spend logs; the update then hits every resetting archived
+-- row.
+UPDATE "LiteLLM_DeletedVerificationToken" k
+SET total_spend = s.sum_spend
+FROM (
+    SELECT k2.token, SUM(l.spend) AS sum_spend
+    FROM (
+        SELECT DISTINCT token
+        FROM "LiteLLM_DeletedVerificationToken"
+        WHERE budget_duration IS NOT NULL
+           OR budget_id IN (
+               SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL
+           )
+    ) k2
+    JOIN "LiteLLM_SpendLogs" l
+      ON l.api_key IN (k2.token, encode(sha256(convert_to(k2.token, 'UTF8')), 'hex'))
+    GROUP BY k2.token
+) s
+WHERE k.token = s.token
+  AND k.total_spend < s.sum_spend
+  AND (k.budget_duration IS NOT NULL
+       OR k.budget_id IN (
+           SELECT budget_id FROM "LiteLLM_BudgetTable" WHERE budget_duration IS NOT NULL
+       ));
+
+-- Verify: this should return 0.
+--   SELECT count(*) FROM "LiteLLM_VerificationToken" WHERE total_spend < spend;

ui/litellm-dashboard/src/components/VirtualKeysPage/keyTableColumns.tsx

@@ -295,7 +295,7 @@ export const getKeyTableColumns = ({
     header: () => (
       <InfoHeader
         label="Lifetime Spend"
-        tooltip="Cumulative spend across every budget period. Budget resets do not touch this value. Keys created before this field existed only count spend from then on."
+        tooltip="Cumulative spend across every budget period. Budget resets do not touch this value. Lifetime tracking started with LiteLLM v1.103.0 on September 19, 2026, so keys created earlier only count spend since that upgrade."
       />
     ),
     size: 130,

ui/litellm-dashboard/src/components/templates/key_info_view.test.tsx

@@ -7,7 +7,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
 import { KeyResponse, Team } from "../key_team_helpers/key_list";
 import { keyDeleteCall, keyUpdateCall } from "../networking";
 import { QueryClient } from "@tanstack/react-query";
-import KeyInfoView from "./key_info_view";
+import KeyInfoView, { needsLifetimeSpendBackfill } from "./key_info_view";
 
 const editViewMocks = vi.hoisted(() => ({
   onSubmit: undefined as ((v: Record<string, any>) => Promise<void>) | undefined,
@@ -290,6 +290,58 @@ describe("KeyInfoView", () => {
     expect(screen.getByTestId("key-lifetime-spend")).toHaveTextContent("Lifetime spend: $340.5000");
   });
 
+  it("shows the backfill hint when lifetime spend trails the period spend", async () => {
+    vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock);
+
+    renderWithProviders(
+      <KeyInfoView
+        keyData={{ ...MOCK_KEY_DATA, spend: 10, total_spend: 4 }}
+        onClose={() => {}}
+        keyId={"test-key-id"}
+        onKeyDataUpdate={() => {}}
+        teams={[]}
+      />,
+    );
+
+    expect(await screen.findByTestId("key-lifetime-spend-backfill-hint")).toBeInTheDocument();
+    expect(screen.getByRole("button", { name: /lifetime spend is below/i })).toBeInTheDocument();
+    expect(screen.getByTestId("key-lifetime-spend")).toHaveTextContent("Lifetime spend: $4.0000");
+  });
+
+  it("hides the backfill hint when lifetime spend covers the period spend", async () => {
+    vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock);
+
+    renderWithProviders(
+      <KeyInfoView
+        keyData={{ ...MOCK_KEY_DATA, spend: 0.25, total_spend: 340.5 }}
+        onClose={() => {}}
+        keyId={"test-key-id"}
+        onKeyDataUpdate={() => {}}
+        teams={[]}
+      />,
+    );
+
+    expect(await screen.findByTestId("key-lifetime-spend")).toBeInTheDocument();
+    expect(screen.queryByTestId("key-lifetime-spend-backfill-hint")).not.toBeInTheDocument();
+  });
+
+  describe("needsLifetimeSpendBackfill", () => {
+    it("returns true when total spend is below the period spend", () => {
+      expect(needsLifetimeSpendBackfill(10, 4)).toBe(true);
+    });
+
+    it("returns false when total spend equals or exceeds the period spend", () => {
+      expect(needsLifetimeSpendBackfill(10, 10)).toBe(false);
+      expect(needsLifetimeSpendBackfill(10, 12)).toBe(false);
+    });
+
+    it("treats a missing total spend as zero", () => {
+      expect(needsLifetimeSpendBackfill(10, null)).toBe(true);
+      expect(needsLifetimeSpendBackfill(10, undefined)).toBe(true);
+      expect(needsLifetimeSpendBackfill(0, null)).toBe(false);
+    });
+  });
+
   it("should render the key's saved router fallbacks", async () => {
     vi.mocked(useAuthorized).mockReturnValue(baseUseAuthorizedMock);
 

ui/litellm-dashboard/src/components/templates/key_info_view.tsx

@@ -6,11 +6,12 @@ import useTeams from "@/app/(dashboard)/hooks/useTeams";
 import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations";
 import { formatNumberWithCommas } from "@/utils/dataUtils";
 import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils";
-import { ArrowLeft } from "lucide-react";
+import { ArrowLeft, Info } from "lucide-react";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Card } from "@/components/ui/card";
 import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
+import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { EntityLink } from "@/components/shared/EntityLink";
 import { modelGroupHref, teamDetailHref } from "@/utils/entityLinks";
@@ -49,6 +50,10 @@ import { parseErrorMessage } from "../shared/errorUtils";
 import { InheritedBudgetHint, inheritedBudgetGates, keyOwnerBudgetSource } from "../shared/InheritedBudgetHint";
 import { KeyEditView } from "./key_edit_view";
 
+export function needsLifetimeSpendBackfill(spend: number, totalSpend: number | null | undefined): boolean {
+  return (totalSpend ?? 0) < spend;
+}
+
 interface KeyInfoViewProps {
   keyId: string;
   onClose: () => void;
@@ -682,6 +687,26 @@ export default function KeyInfoView({
                   )}
                   <p className="text-sm mt-2" data-testid="key-lifetime-spend">
                     Lifetime spend: ${formatNumberWithCommas(currentKeyData.total_spend ?? 0, 4)}
+                    {needsLifetimeSpendBackfill(currentKeyData.spend, currentKeyData.total_spend) && (
+                      <HoverCard>
+                        <HoverCardTrigger
+                          render={
+                            <button
+                              type="button"
+                              aria-label="Why lifetime spend is below current spend"
+                              className="inline-flex align-middle ml-1 cursor-help"
+                              data-testid="key-lifetime-spend-backfill-hint"
+                            />
+                          }
+                        >
+                          <Info className="size-3 text-muted-foreground" />
+                        </HoverCardTrigger>
+                        <HoverCardContent className="w-80">
+                          Lifetime tracking started with LiteLLM v1.103.0 on September 19, 2026 and was not backfilled,
+                          so this key&apos;s lifetime spend only counts usage since that upgrade.
+                        </HoverCardContent>
+                      </HoverCard>
+                    )}
                   </p>
                 </div>
               </Card>