From d16a51a9698c71797f1bc8fefc93100e5d24f178 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 19 Aug 2026 18:17:58 -0700 Subject: [PATCH] fix(ui): tell per-model budget rows apart, and stop calling a throttle a block Per-model caps now report one row per request model, so a cap reachable under two names arrives twice with the same label. The scope cell shows the request model whose counter each row measures, or the two rows read as duplicates. A key that opted into throttle_on_budget_exceeded was rendered as blocking, with a "Blocks at" threshold and a red exceeded badge, when going over actually slows requests instead of rejecting them. Such a row is never the cause of a denial. Deadness is now read from the caveat code alone. Severity no longer tracks it in either direction, since dead codes ship under both values, so an unclassified code is assumed live: calling a row dead when it is not invites dismissing the budget that actually stopped the request. --- .../templates/KeyBudgetsTableColumns.test.ts | 49 ++++++++-- .../templates/KeyBudgetsTableColumns.tsx | 70 ++++++++++---- .../key_info_view.budgets_tab.test.tsx | 92 ++++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 29 ++++-- 4 files changed, 203 insertions(+), 37 deletions(-) diff --git a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts index 43a480193dd..db2013d9e35 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts +++ b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type { KeyBudgetEntry, KeyBudgetNote } from "@/app/(dashboard)/hooks/keys/useKeyBudgets"; -import { budgetThresholdRule, cannotTrip, isBlockingRow, rowRank } from "./KeyBudgetsTableColumns"; +import { budgetThresholdRule, cannotTrip, isBlockingRow, isThrottled, rowRank } from "./KeyBudgetsTableColumns"; const UNCONFIGURED_BUDGET = { scope: "key", @@ -23,7 +23,7 @@ const UNCONFIGURED_BUDGET = { const PROJECT_DEAD_NOTE = { code: "project_spend_not_tracked", - severity: "info", + severity: "warning", text: "project spend is never incremented today, so this budget cannot trip", } as const; @@ -35,10 +35,16 @@ const ALERT_ONLY_NOTE = { const ROLLING_NOTE = { code: "rolling_window", - severity: "warning", + severity: "info", text: "rolling window", } as const; +const THROTTLE_NOTE = { + code: "throttled_instead_of_blocked", + severity: "warning", + text: "this key opted into throttle_on_budget_exceeded, so exceeding it slows requests instead of blocking", +} as const; + // Tagged info by the server, but it scopes which requests the row applies to rather than killing it. const END_USER_ROUTE_NOTE = { code: "end_user_route_only", @@ -157,11 +163,18 @@ describe("cannotTrip", () => { expect(cannotTrip(endUser)).toBe(false); }); - it("falls back to severity for a code this build predates, so a newer server still renders sanely", () => { + it("ignores severity entirely, since dead codes ship under both values", () => { + expect(PROJECT_DEAD_NOTE.severity).toBe("warning"); + expect(cannotTrip(INERT)).toBe(true); + expect(ROLLING_NOTE.severity).toBe("info"); + expect(cannotTrip(WARNED)).toBe(false); + }); + + it("assumes a code this build predates is live, so an unknown caveat never hides a real blocker", () => { const future = { code: "some_code_added_later", severity: "info", text: "…" } as unknown as KeyBudgetNote; - const benign = { ...future, severity: "warning" } as KeyBudgetNote; - expect(cannotTrip({ ...HEALTHY, notes: [future] })).toBe(true); - expect(cannotTrip({ ...HEALTHY, notes: [benign] })).toBe(false); + const louder = { ...future, severity: "warning" } as KeyBudgetNote; + expect(cannotTrip({ ...HEALTHY, notes: [future] })).toBe(false); + expect(cannotTrip({ ...HEALTHY, notes: [louder] })).toBe(false); }); it("branches on code and severity rather than on wording, so text is free to be reworded", () => { @@ -173,6 +186,28 @@ describe("cannotTrip", () => { }); }); +describe("isThrottled", () => { + it("never blames a throttled budget for a denial, because going over slows rather than rejects", () => { + const throttledOver: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + enforcement: "hard", + max_budget: 100, + spend: 140, + remaining: -40, + status: "exceeded", + notes: [THROTTLE_NOTE], + }; + expect(isThrottled(throttledOver)).toBe(true); + expect(isBlockingRow(throttledOver)).toBe(false); + expect(budgetThresholdRule(throttledOver)).toBe("Throttles at ≥ $100.00"); + }); + + it("still says a plain hard budget blocks", () => { + expect(isThrottled(TEAM_MEMBER_AT_LIMIT)).toBe(false); + expect(budgetThresholdRule(TEAM_MEMBER_AT_LIMIT)).toBe("Blocks at ≥ $50.00"); + }); +}); + describe("isBlockingRow", () => { it("counts only a hard budget that is over as blocking", () => { expect(isBlockingRow(BLOCKING)).toBe(true); diff --git a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx index 1728d9ba1f7..d5a83f7a5cb 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx @@ -33,11 +33,11 @@ const isAlertOnly = (entry: KeyBudgetEntry): boolean => entry.enforcement === "s /** * Whether a note means the row is dead: it cannot reject a request no matter what the numbers say. - * Keyed by `code` rather than by `severity` because severity does not track this reliably, and - * exhaustive over the union so a code added server-side fails this build until it is classified. - * `alert_only` is not dead, it restates the `enforcement` column and a soft budget that is over - * still outranks healthy rows. `end_user_route_only` is not dead either, it scopes which requests - * the row applies to. + * Deadness is a property of the code alone. Severity does not imply it in either direction, since + * dead codes appear under both values, so an unclassified code is assumed live: calling a row dead + * when it is not invites dismissing the budget that actually stopped the request, which is the one + * failure this table exists to prevent. Exhaustive over the union, so a code added server-side + * fails this build until someone classifies it. */ const CODE_KILLS_ROW: Readonly> = { alert_only: false, @@ -53,16 +53,21 @@ const CODE_KILLS_ROW: Readonly> = { user_budget_not_applied_to_team_key: true, }; -/** Severity is the fallback for a code this build predates, never the primary signal. */ const noteKillsRow = (note: KeyBudgetNote): boolean => { const classified: boolean | undefined = CODE_KILLS_ROW[note.code]; - return classified ?? note.severity === "info"; + return classified ?? false; }; export const cannotTrip = (entry: KeyBudgetEntry): boolean => entry.notes.some(noteKillsRow); -export const isBlockingRow = (entry: KeyBudgetEntry): boolean => - entry.status === "exceeded" && !isAlertOnly(entry) && !cannotTrip(entry); +/** Exceeding a throttled budget slows requests rather than rejecting them, so it never denies one. */ +export const isThrottled = (entry: KeyBudgetEntry): boolean => + entry.notes.some((note) => note.code === "throttled_instead_of_blocked"); + +/** Whether going over this budget rejects a request, as opposed to alerting, throttling or nothing. */ +const canDeny = (entry: KeyBudgetEntry): boolean => !isAlertOnly(entry) && !cannotTrip(entry) && !isThrottled(entry); + +export const isBlockingRow = (entry: KeyBudgetEntry): boolean => entry.status === "exceeded" && canDeny(entry); /** Ascending relevance to "what stopped my request", so a row that cannot answer it sorts last. */ export const rowRank = (entry: KeyBudgetEntry): number => { @@ -89,23 +94,33 @@ const COMPARISON_GLYPH: Record = { ">=": "≥", ">": ">" }; * to `>`. So this reads `comparison` off each row rather than assuming a constant per scope. Two * rows can show identical numbers and opposite statuses, so state the threshold each one enforces. */ +const thresholdVerb = (entry: KeyBudgetEntry): string => { + if (isAlertOnly(entry)) return "Alerts"; + return isThrottled(entry) ? "Throttles" : "Blocks"; +}; + export const budgetThresholdRule = (entry: KeyBudgetEntry): string | null => { if (entry.max_budget == null) return null; const threshold = `${COMPARISON_GLYPH[entry.comparison] ?? entry.comparison} $${formatNumberWithCommas(entry.max_budget, 2)}`; - return isAlertOnly(entry) ? `Alerts at ${threshold}` : `Blocks at ${threshold}`; + return `${thresholdVerb(entry)} at ${threshold}`; }; const statusPresentation = (entry: KeyBudgetEntry): { tone: StatusTone; label: string } => { if (entry.status === "unlimited") return { tone: "neutral", label: "Unlimited" }; if (cannotTrip(entry)) return { tone: "neutral", label: "Cannot trip" }; if (entry.status !== "exceeded") return { tone: "success", label: "Within budget" }; - return isAlertOnly(entry) - ? { tone: "warning", label: "Exceeded (alert only)" } + if (isAlertOnly(entry)) return { tone: "warning", label: "Exceeded (alert only)" }; + return isThrottled(entry) + ? { tone: "warning", label: "Exceeded (throttling)" } : { tone: "error", label: "Exceeded" }; }; function ScopeCell({ entry }: { entry: KeyBudgetEntry }) { const entity = entry.entity_label || entry.entity_id; + // Per-model rows split one cap across every request model that routes onto it, so `entity_id` is + // what tells two rows apart while `entity_label` repeats the cap. Showing only the label would + // render them as duplicates. + const measured = entry.entity_label && entry.entity_id !== entry.entity_label ? entry.entity_id : null; return (
)} + {measured && ( + + {measured} + + )} {entry.notes.map((note) => ( - ) : ( + if (isAlertOnly(entry)) { + return ( + + ); + } + if (isThrottled(entry)) { + return ( + + ); + } + return ( ); } diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx index 7349fd7b471..fc1c63a2dbb 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx @@ -139,7 +139,7 @@ const ALERT_ONLY_NOTE = { const PROJECT_DEAD_NOTE = { code: "project_spend_not_tracked", - severity: "info", + severity: "warning", text: "project spend is never incremented today, so this budget cannot trip", } as const; @@ -208,7 +208,7 @@ const ORG_UNCONFIGURED: KeyBudgetEntry = { const WORST_CASE_NOTES = [ { code: "reservation_blocks_at_limit", - severity: "warning", + severity: "info", text: "the reservation layer blocks this scope as soon as spend reaches the limit, " + "before the read-time check would trip", @@ -435,6 +435,94 @@ describe("KeyInfoView Budgets tab", () => { expect(panel.getByText(LONGEST_SINGLE_NOTE.text)).toBeInTheDocument(); }); + it("keeps two per-model rows on one cap apart by the request model each measures", async () => { + const direct: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + scope: "key_model", + entity_type: "key", + entity_id: "claude-opus-5", + entity_label: "claude-opus-5", + max_budget: 40, + spend: 5, + remaining: 35, + comparison: ">", + source: "key.model_max_budget[claude-opus-5]", + status: "ok", + }; + const routed: KeyBudgetEntry = { + ...direct, + entity_id: "bedrock/claude-opus-5", + spend: 38, + remaining: 2, + }; + mockBudgets([direct, routed]); + const panel = await renderAndOpenBudgetsTab(); + + const routedRow = rowFor(panel, "bedrock/claude-opus-5"); + expect(routedRow).toHaveTextContent("$38.0000 of $40.00"); + // The cap is repeated on both rows, so it cannot be what tells them apart. + expect(panel.getAllByText("claude-opus-5")).toHaveLength(2); + expect(routedRow).toHaveTextContent("claude-opus-5"); + + const [, ...dataRows] = panel.getAllByRole("row"); + expect(dataRows).toHaveLength(2); + expect(dataRows.filter((row) => row.textContent?.includes("bedrock/claude-opus-5"))).toHaveLength(1); + }); + + it("renders notes in the order the server sent them, most specific to these numbers first", async () => { + const endUser: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + scope: "end_user", + entity_type: "end_user", + entity_id: "customer-42", + entity_label: "customer-42", + max_budget: 25, + spend: 25, + remaining: 0, + source: "budget_table:b-end-user", + status: "exceeded", + notes: [...WORST_CASE_NOTES], + }; + mockBudgets([endUser]); + const panel = await renderAndOpenBudgetsTab(); + + const texts = WORST_CASE_NOTES.map((note) => note.text); + const rendered = texts.map((text) => panel.getByText(text)); + const positions = rendered.map((node) => Array.from(node.parentElement?.children ?? []).indexOf(node)); + expect(positions).toStrictEqual([...positions].sort((a, b) => a - b)); + // Server order is meaningful: the reservation note explains the comparison the row renders. + expect(positions[0]).toBeLessThan(positions[2]); + }); + + it("says a throttled budget slows requests rather than claiming it blocks them", async () => { + const throttled: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + entity_id: "ci-runner", + entity_label: "ci-runner", + max_budget: 100, + spend: 140, + remaining: -40, + source: "key.max_budget", + status: "exceeded", + notes: [ + { + code: "throttled_instead_of_blocked", + severity: "warning", + text: "this key opted into throttle_on_budget_exceeded, so exceeding it slows requests instead of blocking", + }, + ], + }; + mockBudgets([throttled]); + const panel = await renderAndOpenBudgetsTab(); + + const row = rowFor(panel, "ci-runner"); + expect(within(row).getByText("Throttles requests")).toBeInTheDocument(); + expect(within(row).queryByText("Blocks requests")).not.toBeInTheDocument(); + expect(row).toHaveTextContent("Throttles at ≥ $100.00"); + expect(within(row).getByText("Exceeded (throttling)")).toBeInTheDocument(); + expect(within(row).queryByTestId("key-budget-blocking")).not.toBeInTheDocument(); + }); + it("marks a budget that structurally cannot trip and sinks it below every live row", async () => { const dead: KeyBudgetEntry = { ...UNCONFIGURED_BUDGET, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7d2f1809166..d8944633c64 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -6818,8 +6818,9 @@ export interface paths { * can be ruled out without opening every object. * * Parameters: - * - key_id: str | None (path parameter) - The key to inspect. Accepts the plaintext key or its - * hash. Defaults to the key in the Authorization header when omitted (`GET /key/budgets`). + * - key_id: str | None (path parameter) - The hash of the key to inspect. The key itself is + * rejected here, because a URL path reaches access logs, tracing spans and error-logging + * callbacks. Defaults to the key in the Authorization header when omitted (`GET /key/budgets`). * - end_user_id: str | None (query parameter) - Also report the budgets that would apply to this * end user. Omitted end users produce no `end_user` rows, because nothing binds an end user to * a key outside a request. Proxy admins only, since end users are a proxy-global namespace with @@ -6846,11 +6847,13 @@ export interface paths { * - source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:` * - status: str - `unlimited`, `ok` or `exceeded` * - notes: list - Caveats worth knowing before trusting the row, each with a stable `code` - * to branch on, a `severity` of `info` or `warning`, and human-facing `text` + * to branch on, a `severity` of `info` or `warning` for codes a client does not know yet, + * and human-facing `text` that is free to be reworded. Ordered most to least specific to + * this row's numbers, and empty rather than null when there is nothing to say * * Example Curl: * ``` - * curl -X GET "http://0.0.0.0:4000/key/sk-test-example-key-123/budgets" -H "Authorization: Bearer sk-1234" + * curl -X GET "http://0.0.0.0:4000/key/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2/budgets" -H "Authorization: Bearer sk-1234" * ``` * * Example Curl - the budgets on the calling key itself @@ -7512,8 +7515,9 @@ export interface paths { * can be ruled out without opening every object. * * Parameters: - * - key_id: str | None (path parameter) - The key to inspect. Accepts the plaintext key or its - * hash. Defaults to the key in the Authorization header when omitted (`GET /key/budgets`). + * - key_id: str | None (path parameter) - The hash of the key to inspect. The key itself is + * rejected here, because a URL path reaches access logs, tracing spans and error-logging + * callbacks. Defaults to the key in the Authorization header when omitted (`GET /key/budgets`). * - end_user_id: str | None (query parameter) - Also report the budgets that would apply to this * end user. Omitted end users produce no `end_user` rows, because nothing binds an end user to * a key outside a request. Proxy admins only, since end users are a proxy-global namespace with @@ -7540,11 +7544,13 @@ export interface paths { * - source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:` * - status: str - `unlimited`, `ok` or `exceeded` * - notes: list - Caveats worth knowing before trusting the row, each with a stable `code` - * to branch on, a `severity` of `info` or `warning`, and human-facing `text` + * to branch on, a `severity` of `info` or `warning` for codes a client does not know yet, + * and human-facing `text` that is free to be reworded. Ordered most to least specific to + * this row's numbers, and empty rather than null when there is nothing to say * * Example Curl: * ``` - * curl -X GET "http://0.0.0.0:4000/key/sk-test-example-key-123/budgets" -H "Authorization: Bearer sk-1234" + * curl -X GET "http://0.0.0.0:4000/key/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2/budgets" -H "Authorization: Bearer sk-1234" * ``` * * Example Curl - the budgets on the calling key itself @@ -26392,7 +26398,12 @@ export interface components { }; /** * KeyBudgetNote - * @description One caveat about a budget row. Branch on ``code``; ``text`` is free to be reworded. + * @description One caveat about a budget row. + * + * ``code`` is the contract: map it to whatever treatment the caveat deserves. ``text`` is free to be + * reworded and must not be matched on. ``severity`` exists for the code a client has not been taught + * yet, since this union grows: ``warning`` means the row's numbers may be incomplete or read as + * something they are not, and ``info`` means they are accurate and the note is only context. */ KeyBudgetNote: { /**