diff --git a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts index 43a480193dd..db2013d9e35 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts +++ b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import type { KeyBudgetEntry, KeyBudgetNote } from "@/app/(dashboard)/hooks/keys/useKeyBudgets"; -import { budgetThresholdRule, cannotTrip, isBlockingRow, rowRank } from "./KeyBudgetsTableColumns"; +import { budgetThresholdRule, cannotTrip, isBlockingRow, isThrottled, rowRank } from "./KeyBudgetsTableColumns"; const UNCONFIGURED_BUDGET = { scope: "key", @@ -23,7 +23,7 @@ const UNCONFIGURED_BUDGET = { const PROJECT_DEAD_NOTE = { code: "project_spend_not_tracked", - severity: "info", + severity: "warning", text: "project spend is never incremented today, so this budget cannot trip", } as const; @@ -35,10 +35,16 @@ const ALERT_ONLY_NOTE = { const ROLLING_NOTE = { code: "rolling_window", - severity: "warning", + severity: "info", text: "rolling window", } as const; +const THROTTLE_NOTE = { + code: "throttled_instead_of_blocked", + severity: "warning", + text: "this key opted into throttle_on_budget_exceeded, so exceeding it slows requests instead of blocking", +} as const; + // Tagged info by the server, but it scopes which requests the row applies to rather than killing it. const END_USER_ROUTE_NOTE = { code: "end_user_route_only", @@ -157,11 +163,18 @@ describe("cannotTrip", () => { expect(cannotTrip(endUser)).toBe(false); }); - it("falls back to severity for a code this build predates, so a newer server still renders sanely", () => { + it("ignores severity entirely, since dead codes ship under both values", () => { + expect(PROJECT_DEAD_NOTE.severity).toBe("warning"); + expect(cannotTrip(INERT)).toBe(true); + expect(ROLLING_NOTE.severity).toBe("info"); + expect(cannotTrip(WARNED)).toBe(false); + }); + + it("assumes a code this build predates is live, so an unknown caveat never hides a real blocker", () => { const future = { code: "some_code_added_later", severity: "info", text: "…" } as unknown as KeyBudgetNote; - const benign = { ...future, severity: "warning" } as KeyBudgetNote; - expect(cannotTrip({ ...HEALTHY, notes: [future] })).toBe(true); - expect(cannotTrip({ ...HEALTHY, notes: [benign] })).toBe(false); + const louder = { ...future, severity: "warning" } as KeyBudgetNote; + expect(cannotTrip({ ...HEALTHY, notes: [future] })).toBe(false); + expect(cannotTrip({ ...HEALTHY, notes: [louder] })).toBe(false); }); it("branches on code and severity rather than on wording, so text is free to be reworded", () => { @@ -173,6 +186,28 @@ describe("cannotTrip", () => { }); }); +describe("isThrottled", () => { + it("never blames a throttled budget for a denial, because going over slows rather than rejects", () => { + const throttledOver: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + enforcement: "hard", + max_budget: 100, + spend: 140, + remaining: -40, + status: "exceeded", + notes: [THROTTLE_NOTE], + }; + expect(isThrottled(throttledOver)).toBe(true); + expect(isBlockingRow(throttledOver)).toBe(false); + expect(budgetThresholdRule(throttledOver)).toBe("Throttles at ≥ $100.00"); + }); + + it("still says a plain hard budget blocks", () => { + expect(isThrottled(TEAM_MEMBER_AT_LIMIT)).toBe(false); + expect(budgetThresholdRule(TEAM_MEMBER_AT_LIMIT)).toBe("Blocks at ≥ $50.00"); + }); +}); + describe("isBlockingRow", () => { it("counts only a hard budget that is over as blocking", () => { expect(isBlockingRow(BLOCKING)).toBe(true); diff --git a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx index 1728d9ba1f7..d5a83f7a5cb 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.tsx @@ -33,11 +33,11 @@ const isAlertOnly = (entry: KeyBudgetEntry): boolean => entry.enforcement === "s /** * Whether a note means the row is dead: it cannot reject a request no matter what the numbers say. - * Keyed by `code` rather than by `severity` because severity does not track this reliably, and - * exhaustive over the union so a code added server-side fails this build until it is classified. - * `alert_only` is not dead, it restates the `enforcement` column and a soft budget that is over - * still outranks healthy rows. `end_user_route_only` is not dead either, it scopes which requests - * the row applies to. + * Deadness is a property of the code alone. Severity does not imply it in either direction, since + * dead codes appear under both values, so an unclassified code is assumed live: calling a row dead + * when it is not invites dismissing the budget that actually stopped the request, which is the one + * failure this table exists to prevent. Exhaustive over the union, so a code added server-side + * fails this build until someone classifies it. */ const CODE_KILLS_ROW: Readonly> = { alert_only: false, @@ -53,16 +53,21 @@ const CODE_KILLS_ROW: Readonly> = { user_budget_not_applied_to_team_key: true, }; -/** Severity is the fallback for a code this build predates, never the primary signal. */ const noteKillsRow = (note: KeyBudgetNote): boolean => { const classified: boolean | undefined = CODE_KILLS_ROW[note.code]; - return classified ?? note.severity === "info"; + return classified ?? false; }; export const cannotTrip = (entry: KeyBudgetEntry): boolean => entry.notes.some(noteKillsRow); -export const isBlockingRow = (entry: KeyBudgetEntry): boolean => - entry.status === "exceeded" && !isAlertOnly(entry) && !cannotTrip(entry); +/** Exceeding a throttled budget slows requests rather than rejecting them, so it never denies one. */ +export const isThrottled = (entry: KeyBudgetEntry): boolean => + entry.notes.some((note) => note.code === "throttled_instead_of_blocked"); + +/** Whether going over this budget rejects a request, as opposed to alerting, throttling or nothing. */ +const canDeny = (entry: KeyBudgetEntry): boolean => !isAlertOnly(entry) && !cannotTrip(entry) && !isThrottled(entry); + +export const isBlockingRow = (entry: KeyBudgetEntry): boolean => entry.status === "exceeded" && canDeny(entry); /** Ascending relevance to "what stopped my request", so a row that cannot answer it sorts last. */ export const rowRank = (entry: KeyBudgetEntry): number => { @@ -89,23 +94,33 @@ const COMPARISON_GLYPH: Record = { ">=": "≥", ">": ">" }; * to `>`. So this reads `comparison` off each row rather than assuming a constant per scope. Two * rows can show identical numbers and opposite statuses, so state the threshold each one enforces. */ +const thresholdVerb = (entry: KeyBudgetEntry): string => { + if (isAlertOnly(entry)) return "Alerts"; + return isThrottled(entry) ? "Throttles" : "Blocks"; +}; + export const budgetThresholdRule = (entry: KeyBudgetEntry): string | null => { if (entry.max_budget == null) return null; const threshold = `${COMPARISON_GLYPH[entry.comparison] ?? entry.comparison} $${formatNumberWithCommas(entry.max_budget, 2)}`; - return isAlertOnly(entry) ? `Alerts at ${threshold}` : `Blocks at ${threshold}`; + return `${thresholdVerb(entry)} at ${threshold}`; }; const statusPresentation = (entry: KeyBudgetEntry): { tone: StatusTone; label: string } => { if (entry.status === "unlimited") return { tone: "neutral", label: "Unlimited" }; if (cannotTrip(entry)) return { tone: "neutral", label: "Cannot trip" }; if (entry.status !== "exceeded") return { tone: "success", label: "Within budget" }; - return isAlertOnly(entry) - ? { tone: "warning", label: "Exceeded (alert only)" } + if (isAlertOnly(entry)) return { tone: "warning", label: "Exceeded (alert only)" }; + return isThrottled(entry) + ? { tone: "warning", label: "Exceeded (throttling)" } : { tone: "error", label: "Exceeded" }; }; function ScopeCell({ entry }: { entry: KeyBudgetEntry }) { const entity = entry.entity_label || entry.entity_id; + // Per-model rows split one cap across every request model that routes onto it, so `entity_id` is + // what tells two rows apart while `entity_label` repeats the cap. Showing only the label would + // render them as duplicates. + const measured = entry.entity_label && entry.entity_id !== entry.entity_label ? entry.entity_id : null; return (
)} + {measured && ( + + {measured} + + )} {entry.notes.map((note) => ( - ) : ( + if (isAlertOnly(entry)) { + return ( + + ); + } + if (isThrottled(entry)) { + return ( + + ); + } + return ( ); } diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx index 7349fd7b471..fc1c63a2dbb 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx @@ -139,7 +139,7 @@ const ALERT_ONLY_NOTE = { const PROJECT_DEAD_NOTE = { code: "project_spend_not_tracked", - severity: "info", + severity: "warning", text: "project spend is never incremented today, so this budget cannot trip", } as const; @@ -208,7 +208,7 @@ const ORG_UNCONFIGURED: KeyBudgetEntry = { const WORST_CASE_NOTES = [ { code: "reservation_blocks_at_limit", - severity: "warning", + severity: "info", text: "the reservation layer blocks this scope as soon as spend reaches the limit, " + "before the read-time check would trip", @@ -435,6 +435,94 @@ describe("KeyInfoView Budgets tab", () => { expect(panel.getByText(LONGEST_SINGLE_NOTE.text)).toBeInTheDocument(); }); + it("keeps two per-model rows on one cap apart by the request model each measures", async () => { + const direct: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + scope: "key_model", + entity_type: "key", + entity_id: "claude-opus-5", + entity_label: "claude-opus-5", + max_budget: 40, + spend: 5, + remaining: 35, + comparison: ">", + source: "key.model_max_budget[claude-opus-5]", + status: "ok", + }; + const routed: KeyBudgetEntry = { + ...direct, + entity_id: "bedrock/claude-opus-5", + spend: 38, + remaining: 2, + }; + mockBudgets([direct, routed]); + const panel = await renderAndOpenBudgetsTab(); + + const routedRow = rowFor(panel, "bedrock/claude-opus-5"); + expect(routedRow).toHaveTextContent("$38.0000 of $40.00"); + // The cap is repeated on both rows, so it cannot be what tells them apart. + expect(panel.getAllByText("claude-opus-5")).toHaveLength(2); + expect(routedRow).toHaveTextContent("claude-opus-5"); + + const [, ...dataRows] = panel.getAllByRole("row"); + expect(dataRows).toHaveLength(2); + expect(dataRows.filter((row) => row.textContent?.includes("bedrock/claude-opus-5"))).toHaveLength(1); + }); + + it("renders notes in the order the server sent them, most specific to these numbers first", async () => { + const endUser: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + scope: "end_user", + entity_type: "end_user", + entity_id: "customer-42", + entity_label: "customer-42", + max_budget: 25, + spend: 25, + remaining: 0, + source: "budget_table:b-end-user", + status: "exceeded", + notes: [...WORST_CASE_NOTES], + }; + mockBudgets([endUser]); + const panel = await renderAndOpenBudgetsTab(); + + const texts = WORST_CASE_NOTES.map((note) => note.text); + const rendered = texts.map((text) => panel.getByText(text)); + const positions = rendered.map((node) => Array.from(node.parentElement?.children ?? []).indexOf(node)); + expect(positions).toStrictEqual([...positions].sort((a, b) => a - b)); + // Server order is meaningful: the reservation note explains the comparison the row renders. + expect(positions[0]).toBeLessThan(positions[2]); + }); + + it("says a throttled budget slows requests rather than claiming it blocks them", async () => { + const throttled: KeyBudgetEntry = { + ...UNCONFIGURED_BUDGET, + entity_id: "ci-runner", + entity_label: "ci-runner", + max_budget: 100, + spend: 140, + remaining: -40, + source: "key.max_budget", + status: "exceeded", + notes: [ + { + code: "throttled_instead_of_blocked", + severity: "warning", + text: "this key opted into throttle_on_budget_exceeded, so exceeding it slows requests instead of blocking", + }, + ], + }; + mockBudgets([throttled]); + const panel = await renderAndOpenBudgetsTab(); + + const row = rowFor(panel, "ci-runner"); + expect(within(row).getByText("Throttles requests")).toBeInTheDocument(); + expect(within(row).queryByText("Blocks requests")).not.toBeInTheDocument(); + expect(row).toHaveTextContent("Throttles at ≥ $100.00"); + expect(within(row).getByText("Exceeded (throttling)")).toBeInTheDocument(); + expect(within(row).queryByTestId("key-budget-blocking")).not.toBeInTheDocument(); + }); + it("marks a budget that structurally cannot trip and sinks it below every live row", async () => { const dead: KeyBudgetEntry = { ...UNCONFIGURED_BUDGET, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7d2f1809166..d8944633c64 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -6818,8 +6818,9 @@ export interface paths { * can be ruled out without opening every object. * * Parameters: - * - key_id: str | None (path parameter) - The key to inspect. Accepts the plaintext key or its - * hash. Defaults to the key in the Authorization header when omitted (`GET /key/budgets`). + * - key_id: str | None (path parameter) - The hash of the key to inspect. The key itself is + * rejected here, because a URL path reaches access logs, tracing spans and error-logging + * callbacks. Defaults to the key in the Authorization header when omitted (`GET /key/budgets`). * - end_user_id: str | None (query parameter) - Also report the budgets that would apply to this * end user. Omitted end users produce no `end_user` rows, because nothing binds an end user to * a key outside a request. Proxy admins only, since end users are a proxy-global namespace with @@ -6846,11 +6847,13 @@ export interface paths { * - source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:` * - status: str - `unlimited`, `ok` or `exceeded` * - notes: list - Caveats worth knowing before trusting the row, each with a stable `code` - * to branch on, a `severity` of `info` or `warning`, and human-facing `text` + * to branch on, a `severity` of `info` or `warning` for codes a client does not know yet, + * and human-facing `text` that is free to be reworded. Ordered most to least specific to + * this row's numbers, and empty rather than null when there is nothing to say * * Example Curl: * ``` - * curl -X GET "http://0.0.0.0:4000/key/sk-test-example-key-123/budgets" -H "Authorization: Bearer sk-1234" + * curl -X GET "http://0.0.0.0:4000/key/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2/budgets" -H "Authorization: Bearer sk-1234" * ``` * * Example Curl - the budgets on the calling key itself @@ -7512,8 +7515,9 @@ export interface paths { * can be ruled out without opening every object. * * Parameters: - * - key_id: str | None (path parameter) - The key to inspect. Accepts the plaintext key or its - * hash. Defaults to the key in the Authorization header when omitted (`GET /key/budgets`). + * - key_id: str | None (path parameter) - The hash of the key to inspect. The key itself is + * rejected here, because a URL path reaches access logs, tracing spans and error-logging + * callbacks. Defaults to the key in the Authorization header when omitted (`GET /key/budgets`). * - end_user_id: str | None (query parameter) - Also report the budgets that would apply to this * end user. Omitted end users produce no `end_user` rows, because nothing binds an end user to * a key outside a request. Proxy admins only, since end users are a proxy-global namespace with @@ -7540,11 +7544,13 @@ export interface paths { * - source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:` * - status: str - `unlimited`, `ok` or `exceeded` * - notes: list - Caveats worth knowing before trusting the row, each with a stable `code` - * to branch on, a `severity` of `info` or `warning`, and human-facing `text` + * to branch on, a `severity` of `info` or `warning` for codes a client does not know yet, + * and human-facing `text` that is free to be reworded. Ordered most to least specific to + * this row's numbers, and empty rather than null when there is nothing to say * * Example Curl: * ``` - * curl -X GET "http://0.0.0.0:4000/key/sk-test-example-key-123/budgets" -H "Authorization: Bearer sk-1234" + * curl -X GET "http://0.0.0.0:4000/key/a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2/budgets" -H "Authorization: Bearer sk-1234" * ``` * * Example Curl - the budgets on the calling key itself @@ -26392,7 +26398,12 @@ export interface components { }; /** * KeyBudgetNote - * @description One caveat about a budget row. Branch on ``code``; ``text`` is free to be reworded. + * @description One caveat about a budget row. + * + * ``code`` is the contract: map it to whatever treatment the caveat deserves. ``text`` is free to be + * reworded and must not be matched on. ``severity`` exists for the code a client has not been taught + * yet, since this union grows: ``warning`` means the row's numbers may be incomplete or read as + * something they are not, and ``info`` means they are accurate and the note is only context. */ KeyBudgetNote: { /**