diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyBudgets.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyBudgets.ts index 8620fead52e..c5c8fdd472b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyBudgets.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/keys/useKeyBudgets.ts @@ -4,6 +4,8 @@ import type { components } from "@/lib/http/schema"; export type KeyBudgetsResponse = components["schemas"]["KeyBudgetsResponse"]; export type KeyBudgetEntry = KeyBudgetsResponse["budgets"][number]; +export type KeyBudgetNote = KeyBudgetEntry["notes"][number]; +export type KeyBudgetNoteCode = KeyBudgetNote["code"]; export const useKeyBudgets = (keyId: string | undefined) => { const { accessToken } = useAuthorized(); diff --git a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTable.tsx b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTable.tsx index f031f5d90e5..9e4b36eb3be 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTable.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTable.tsx @@ -7,11 +7,11 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { DataTable } from "@/components/shared/DataTable"; import { parseErrorMessage } from "../shared/errorUtils"; -import { getKeyBudgetsTableColumns, isBlockingRow, severityRank } from "./KeyBudgetsTableColumns"; +import { getKeyBudgetsTableColumns, isBlockingRow, rowRank } from "./KeyBudgetsTableColumns"; function BudgetRows({ budgets, isLoading }: { budgets: readonly KeyBudgetEntry[]; isLoading: boolean }) { const columns = useMemo(() => getKeyBudgetsTableColumns(), []); - const rows = useMemo(() => [...budgets].sort((a, b) => severityRank(a) - severityRank(b)), [budgets]); + const rows = useMemo(() => [...budgets].sort((a, b) => rowRank(a) - rowRank(b)), [budgets]); return ( ", max_budget: 0.1 }; const BLOCKING: KeyBudgetEntry = { ...UNCONFIGURED_BUDGET, status: "exceeded", enforcement: "hard" }; -const ALERT_ONLY: KeyBudgetEntry = { ...UNCONFIGURED_BUDGET, status: "exceeded", enforcement: "soft" }; +const ALERT_ONLY: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + status: "exceeded", + enforcement: "soft", + notes: [ALERT_ONLY_NOTE], +}; const HEALTHY: KeyBudgetEntry = { ...UNCONFIGURED_BUDGET, status: "ok", enforcement: "hard" }; +const INERT: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + scope: "project", + status: "ok", + max_budget: 300, + spend: 0, + remaining: 300, + notes: [PROJECT_DEAD_NOTE], +}; +const WARNED: KeyBudgetEntry = { ...HEALTHY, notes: [ROLLING_NOTE] }; describe("budgetThresholdRule", () => { it("marks an inclusive scope as blocking at the limit, so spend equal to it is already denied", () => { @@ -97,6 +138,41 @@ describe("budgetThresholdRule", () => { }); }); +describe("cannotTrip", () => { + it("treats an info note as proof the row is dead", () => { + expect(cannotTrip(INERT)).toBe(true); + }); + + it("does not call a soft budget dead, since alert_only only restates the enforcement column", () => { + expect(cannotTrip(ALERT_ONLY)).toBe(false); + }); + + it("leaves a warning note trippable, because it qualifies the number rather than killing it", () => { + expect(cannotTrip(WARNED)).toBe(false); + }); + + it("keeps an end_user row alive even though the server tags its route caveat as info", () => { + const endUser: KeyBudgetEntry = { ...HEALTHY, scope: "end_user", notes: [END_USER_ROUTE_NOTE] }; + expect(END_USER_ROUTE_NOTE.severity).toBe("info"); + expect(cannotTrip(endUser)).toBe(false); + }); + + it("falls back to severity for a code this build predates, so a newer server still renders sanely", () => { + const future = { code: "some_code_added_later", severity: "info", text: "…" } as unknown as KeyBudgetNote; + const benign = { ...future, severity: "warning" } as KeyBudgetNote; + expect(cannotTrip({ ...HEALTHY, notes: [future] })).toBe(true); + expect(cannotTrip({ ...HEALTHY, notes: [benign] })).toBe(false); + }); + + it("branches on code and severity rather than on wording, so text is free to be reworded", () => { + const reworded: KeyBudgetEntry = { + ...INERT, + notes: [{ ...PROJECT_DEAD_NOTE, text: "totally different prose that never mentions tripping" }], + }; + expect(cannotTrip(reworded)).toBe(true); + }); +}); + describe("isBlockingRow", () => { it("counts only a hard budget that is over as blocking", () => { expect(isBlockingRow(BLOCKING)).toBe(true); @@ -104,12 +180,25 @@ describe("isBlockingRow", () => { expect(isBlockingRow(HEALTHY)).toBe(false); expect(isBlockingRow(UNCONFIGURED_BUDGET)).toBe(false); }); -}); -describe("severityRank", () => { - it("ranks a blocking budget above an alert-only one, and both above healthy and unlimited", () => { - expect(severityRank(BLOCKING)).toBeLessThan(severityRank(ALERT_ONLY)); - expect(severityRank(ALERT_ONLY)).toBeLessThan(severityRank(HEALTHY)); - expect(severityRank(HEALTHY)).toBeLessThan(severityRank(UNCONFIGURED_BUDGET)); + it("never blames a budget that structurally cannot trip, however over it looks", () => { + const overButDead: KeyBudgetEntry = { ...INERT, spend: 900, remaining: -600, status: "exceeded" }; + expect(overButDead.status).toBe("exceeded"); + expect(isBlockingRow(overButDead)).toBe(false); + }); +}); + +describe("rowRank", () => { + it("ranks a blocking budget above an alert-only one, and both above healthy and unlimited", () => { + expect(rowRank(BLOCKING)).toBeLessThan(rowRank(ALERT_ONLY)); + expect(rowRank(ALERT_ONLY)).toBeLessThan(rowRank(HEALTHY)); + expect(rowRank(HEALTHY)).toBeLessThan(rowRank(UNCONFIGURED_BUDGET)); + }); + + it("sinks a row that cannot trip below every row that can, since it never stopped anything", () => { + expect(rowRank(INERT)).toBeGreaterThan(rowRank(BLOCKING)); + expect(rowRank(INERT)).toBeGreaterThan(rowRank(ALERT_ONLY)); + expect(rowRank(INERT)).toBeGreaterThan(rowRank(HEALTHY)); + expect(rowRank(INERT)).toBeGreaterThan(rowRank(UNCONFIGURED_BUDGET)); }); }); diff --git a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx index 8491f7cc675..1728d9ba1f7 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx @@ -2,7 +2,7 @@ import type { ColumnDef } from "@tanstack/react-table"; -import type { KeyBudgetEntry } from "@/app/(dashboard)/hooks/keys/useKeyBudgets"; +import type { KeyBudgetEntry, KeyBudgetNote, KeyBudgetNoteCode } from "@/app/(dashboard)/hooks/keys/useKeyBudgets"; import { CellTooltip, DateCell, @@ -31,12 +31,55 @@ const SCOPE_LABELS: Record = { const isAlertOnly = (entry: KeyBudgetEntry): boolean => entry.enforcement === "soft"; -export const isBlockingRow = (entry: KeyBudgetEntry): boolean => entry.status === "exceeded" && !isAlertOnly(entry); +/** + * Whether a note means the row is dead: it cannot reject a request no matter what the numbers say. + * Keyed by `code` rather than by `severity` because severity does not track this reliably, and + * exhaustive over the union so a code added server-side fails this build until it is classified. + * `alert_only` is not dead, it restates the `enforcement` column and a soft budget that is over + * still outranks healthy rows. `end_user_route_only` is not dead either, it scopes which requests + * the row applies to. + */ +const CODE_KILLS_ROW: Readonly> = { + alert_only: false, + custom_auth_may_override_end_user_cap: false, + end_user_route_only: false, + model_budget_fails_open: true, + per_model_counters: false, + project_spend_not_tracked: true, + request_tags_add_budgets: false, + reservation_blocks_at_limit: false, + rolling_window: false, + throttled_instead_of_blocked: false, + user_budget_not_applied_to_team_key: true, +}; -const STATUS_ORDER: Record = { exceeded: 0, ok: 1, unlimited: 2 }; +/** Severity is the fallback for a code this build predates, never the primary signal. */ +const noteKillsRow = (note: KeyBudgetNote): boolean => { + const classified: boolean | undefined = CODE_KILLS_ROW[note.code]; + return classified ?? note.severity === "info"; +}; -export const severityRank = (entry: KeyBudgetEntry): number => - (STATUS_ORDER[entry.status] ?? STATUS_ORDER.unlimited) * 2 + (isAlertOnly(entry) ? 1 : 0); +export const cannotTrip = (entry: KeyBudgetEntry): boolean => entry.notes.some(noteKillsRow); + +export const isBlockingRow = (entry: KeyBudgetEntry): boolean => + entry.status === "exceeded" && !isAlertOnly(entry) && !cannotTrip(entry); + +/** Ascending relevance to "what stopped my request", so a row that cannot answer it sorts last. */ +export const rowRank = (entry: KeyBudgetEntry): number => { + if (entry.status === "unlimited") return 3; + if (cannotTrip(entry)) return 4; + if (isBlockingRow(entry)) return 0; + if (entry.status === "exceeded") return 1; + return 2; +}; + +/** + * Only `live` and `no_counter` carry a number worth drawing. Any other state, including one the + * server adds after this ships, is rendered as unknown rather than as a confident zero: overstating + * a spend is the failure that matters here, and a new state is by definition not the normal one. + */ +const spendIsReadable = (entry: KeyBudgetEntry): boolean => + entry.spend_state === "live" || entry.spend_state === "no_counter"; const COMPARISON_GLYPH: Record = { ">=": "≥", ">": ">" }; @@ -54,6 +97,7 @@ export const budgetThresholdRule = (entry: KeyBudgetEntry): string | null => { const statusPresentation = (entry: KeyBudgetEntry): { tone: StatusTone; label: string } => { if (entry.status === "unlimited") return { tone: "neutral", label: "Unlimited" }; + if (cannotTrip(entry)) return { tone: "neutral", label: "Cannot trip" }; if (entry.status !== "exceeded") return { tone: "success", label: "Within budget" }; return isAlertOnly(entry) ? { tone: "warning", label: "Exceeded (alert only)" } @@ -77,7 +121,16 @@ function ScopeCell({ entry }: { entry: KeyBudgetEntry }) { {entity} )} - {entry.note && {entry.note}} + {entry.notes.map((note) => ( + + {note.text} + + ))} ); } @@ -94,24 +147,19 @@ function EnforcementCell({ entry }: { entry: KeyBudgetEntry }) { ); } -/** - * A null spend means the live counter could not be read, which `SpendBudgetCell` coerces to 0 and - * draws as an empty meter. That renders a failed read as confident full headroom, so branch before - * reaching it: an unknown number must not look like a healthy one. - */ function SpendCell({ entry }: { entry: KeyBudgetEntry }) { const rule = budgetThresholdRule(entry); return (
- {entry.spend == null ? ( + {spendIsReadable(entry) ? ( + + ) : ( Unknown{" "} {entry.max_budget == null ? "· Unlimited" : `of $${formatNumberWithCommas(entry.max_budget, 2)}`} - ) : ( - )} {rule && {rule}}
diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx index 4a978c62923..7349fd7b471 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx @@ -127,9 +127,22 @@ const UNCONFIGURED_BUDGET = { window_start: null, source: "key.max_budget", status: "unlimited", - note: null, + spend_state: "live", + notes: [], } as KeyBudgetEntry; +const ALERT_ONLY_NOTE = { + code: "alert_only", + severity: "info", + text: "alert only, never blocks; compared against recorded spend rather than the live counter", +} as const; + +const PROJECT_DEAD_NOTE = { + code: "project_spend_not_tracked", + severity: "info", + text: "project spend is never incremented today, so this budget cannot trip", +} as const; + const KEY_UNLIMITED: KeyBudgetEntry = { ...UNCONFIGURED_BUDGET, entity_id: "test-token-123", @@ -162,7 +175,7 @@ const TEAM_SOFT_OVER: KeyBudgetEntry = { remaining: -400, source: "budget_table:b-soft", status: "exceeded", - note: "alert only, never blocks; compared against recorded spend rather than the live counter", + notes: [ALERT_ONLY_NOTE], }; const TEAM_MEMBER_BLOCKING: KeyBudgetEntry = { @@ -190,13 +203,36 @@ const ORG_UNCONFIGURED: KeyBudgetEntry = { source: "organization.budget_id", }; -// Longest note the endpoint can emit: 276 characters over three clauses, reachable only on an -// end_user row where reservation tightened the operator and a custom auth callable is configured. -// The reservation clause lands only on team, tag and end_user, so this is a real ceiling. -const WORST_CASE_NOTE = - "the reservation layer blocks this scope as soon as spend reaches the limit, before the read-time " + - "check would trip; only enforced on LLM routes that name this end user; a custom auth callable can " + - "set a request-scoped end user cap that overrides this one and is not visible here"; +// The widest row the endpoint can emit: an end_user budget the reservation layer tightened, on a +// proxy with a custom auth callable. Three notes rather than one sentence, in the server's order. +const WORST_CASE_NOTES = [ + { + code: "reservation_blocks_at_limit", + severity: "warning", + text: + "the reservation layer blocks this scope as soon as spend reaches the limit, " + + "before the read-time check would trip", + }, + { + code: "end_user_route_only", + severity: "info", + text: "only enforced on LLM routes that name this end user", + }, + { + code: "custom_auth_may_override_end_user_cap", + severity: "warning", + text: "a custom auth callable can set a request-scoped end user cap that overrides this one and is not visible here", + }, +] as const; + +// Longest single note the endpoint can emit, and so the real width constraint on the scope column. +const LONGEST_SINGLE_NOTE = { + code: "per_model_counters", + severity: "warning", + text: + "per-model spend is counted separately for every request model that maps onto this cap, " + + "and each counter is compared against it on its own, so the highest is reported", +} as const; const ALL_BUDGETS = [KEY_UNLIMITED, USER_WITHIN_BUDGET, TEAM_SOFT_OVER, TEAM_MEMBER_BLOCKING, ORG_UNCONFIGURED]; @@ -290,10 +326,10 @@ describe("KeyInfoView Budgets tab", () => { entity_label: "prod", max_budget: 1000, spend: null, + spend_state: "unavailable", remaining: null, source: "budget_table:b-tag", status: "ok", - note: "live spend could not be read", }; mockBudgets([unreadable]); const panel = await renderAndOpenBudgetsTab(); @@ -306,7 +342,50 @@ describe("KeyInfoView Budgets tab", () => { expect(row).toHaveTextContent("Blocks at ≥ $1,000.00"); }); - it("renders the longest note the endpoint can emit without dropping any of it", async () => { + it("shows a genuine no-counter-yet zero as $0.00 with its meter, not as unknown", async () => { + const cold: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + scope: "key_model", + entity_type: "key", + entity_label: "claude-opus-5", + max_budget: 40, + spend: null, + spend_state: "no_counter", + remaining: 40, + source: "key.model_max_budget", + status: "ok", + }; + mockBudgets([cold]); + const panel = await renderAndOpenBudgetsTab(); + + const row = rowFor(panel, "claude-opus-5"); + expect(row).toHaveTextContent("$0.00 of $40.00"); + expect(row).not.toHaveTextContent("Unknown"); + expect(within(row).getByRole("meter")).toBeInTheDocument(); + }); + + it("treats a spend state this build predates as unreadable rather than as a confident number", async () => { + const future: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + scope: "team", + entity_type: "team", + entity_label: "Platform", + max_budget: 500, + spend: 20, + remaining: 480, + source: "team.max_budget", + status: "ok", + spend_state: "reconciling" as KeyBudgetEntry["spend_state"], + }; + mockBudgets([future]); + const panel = await renderAndOpenBudgetsTab(); + + const row = rowFor(panel, "Platform"); + expect(row).toHaveTextContent("Unknown of $500.00"); + expect(within(row).queryByRole("meter")).not.toBeInTheDocument(); + }); + + it("renders every note on the widest row as its own line, none of them dropped", async () => { const endUser: KeyBudgetEntry = { ...UNCONFIGURED_BUDGET, scope: "end_user", @@ -318,18 +397,67 @@ describe("KeyInfoView Budgets tab", () => { remaining: 0, source: "budget_table:b-end-user", status: "exceeded", - note: WORST_CASE_NOTE, + notes: [...WORST_CASE_NOTES], }; mockBudgets([endUser]); const panel = await renderAndOpenBudgetsTab(); - const note = panel.getByText(WORST_CASE_NOTE); - expect(note).toBeInTheDocument(); + const rendered = WORST_CASE_NOTES.map((note) => panel.getByText(note.text)); + expect(rendered).toHaveLength(3); + // Separate elements, not one joined blob, so each caveat can carry its own severity. + expect(new Set(rendered).size).toBe(3); + // jsdom has no layout, so nothing here can prove the text is visually unclipped. Asserting the - // absence of the clipping utilities is the only mechanical guard against re-truncating the note. - expect(note).not.toHaveClass("truncate"); - expect(note.className).not.toMatch(/line-clamp|overflow-hidden|whitespace-nowrap/); - expect(note).not.toHaveAttribute("title"); + // absence of the clipping utilities is the only mechanical guard against re-truncating a note. + for (const note of rendered) { + expect(note).not.toHaveClass("truncate"); + expect(note.className).not.toMatch(/line-clamp|overflow-hidden|whitespace-nowrap/); + expect(note).not.toHaveAttribute("title"); + } + }); + + it("renders the longest single note the endpoint can emit without dropping any of it", async () => { + const perModel: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + scope: "key_model", + entity_type: "key", + entity_label: "claude-opus-5", + max_budget: 40, + spend: 12, + remaining: 28, + source: "key.model_max_budget", + status: "ok", + notes: [LONGEST_SINGLE_NOTE], + }; + mockBudgets([perModel]); + const panel = await renderAndOpenBudgetsTab(); + + expect(panel.getByText(LONGEST_SINGLE_NOTE.text)).toBeInTheDocument(); + }); + + it("marks a budget that structurally cannot trip and sinks it below every live row", async () => { + const dead: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + scope: "project", + entity_type: "project", + entity_label: "checkout", + max_budget: 300, + spend: 0, + remaining: 300, + source: "project.budget_id", + status: "ok", + notes: [PROJECT_DEAD_NOTE], + }; + mockBudgets([dead, TEAM_MEMBER_BLOCKING, USER_WITHIN_BUDGET, KEY_UNLIMITED]); + const panel = await renderAndOpenBudgetsTab(); + + const deadRow = rowFor(panel, "checkout"); + expect(within(deadRow).getByText("Cannot trip")).toBeInTheDocument(); + expect(within(deadRow).queryByText("Within budget")).not.toBeInTheDocument(); + expect(deadRow).toHaveTextContent(PROJECT_DEAD_NOTE.text); + + const [, ...dataRows] = panel.getAllByRole("row"); + expect(dataRows[dataRows.length - 1]).toHaveTextContent("checkout"); }); it("explains why two rows on identical numbers get opposite statuses", async () => { diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 77fbb1cb4f7..7d2f1809166 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -6838,12 +6838,15 @@ export interface paths { * but places no limit on it * - spend: float | None - Spend as the enforcing check reads it, from the same cross-pod * counter, not the periodically-synced database column + * - spend_state: str - Whether `spend` is `live`, missing because no counter exists yet + * (`no_counter`), or missing because the read failed (`unavailable`) * - remaining: float | None - `max_budget - spend`, when both are known * - comparison: str - The operator the enforcing check uses, which differs per scope * - budget_duration / budget_reset_at / window_start: When spend next resets to zero * - source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:` * - status: str - `unlimited`, `ok` or `exceeded` - * - note: str | None - A caveat worth knowing before trusting the row + * - notes: list - Caveats worth knowing before trusting the row, each with a stable `code` + * to branch on, a `severity` of `info` or `warning`, and human-facing `text` * * Example Curl: * ``` @@ -7529,12 +7532,15 @@ export interface paths { * but places no limit on it * - spend: float | None - Spend as the enforcing check reads it, from the same cross-pod * counter, not the periodically-synced database column + * - spend_state: str - Whether `spend` is `live`, missing because no counter exists yet + * (`no_counter`), or missing because the read failed (`unavailable`) * - remaining: float | None - `max_budget - spend`, when both are known * - comparison: str - The operator the enforcing check uses, which differs per scope * - budget_duration / budget_reset_at / window_start: When spend next resets to zero * - source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:` * - status: str - `unlimited`, `ok` or `exceeded` - * - note: str | None - A caveat worth knowing before trusting the row + * - notes: list - Caveats worth knowing before trusting the row, each with a stable `code` + * to branch on, a `severity` of `info` or `warning`, and human-facing `text` * * Example Curl: * ``` @@ -26355,8 +26361,11 @@ export interface components { entity_type: components["schemas"]["Litellm_EntityType"]; /** Max Budget */ max_budget?: number | null; - /** Note */ - note?: string | null; + /** + * Notes + * @default [] + */ + notes: components["schemas"]["KeyBudgetNote"][]; /** Remaining */ remaining?: number | null; /** @@ -26368,6 +26377,11 @@ export interface components { source: string; /** Spend */ spend?: number | null; + /** + * Spend State + * @enum {string} + */ + spend_state: "live" | "no_counter" | "unavailable"; /** * Status * @enum {string} @@ -26376,6 +26390,24 @@ export interface components { /** Window Start */ window_start?: string | null; }; + /** + * KeyBudgetNote + * @description One caveat about a budget row. Branch on ``code``; ``text`` is free to be reworded. + */ + KeyBudgetNote: { + /** + * Code + * @enum {string} + */ + code: "alert_only" | "custom_auth_may_override_end_user_cap" | "end_user_route_only" | "model_budget_fails_open" | "per_model_counters" | "project_spend_not_tracked" | "request_tags_add_budgets" | "reservation_blocks_at_limit" | "rolling_window" | "throttled_instead_of_blocked" | "user_budget_not_applied_to_team_key"; + /** + * Severity + * @enum {string} + */ + severity: "info" | "warning"; + /** Text */ + text: string; + }; /** * KeyBudgetsResponse * @description Every budget that applies to one key, including the ones left unconfigured.