From ed7cee5d50b9101d215f639ecbcb505361b76a00 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 19 Aug 2026 18:21:57 -0700 Subject: [PATCH] test(ui): pin per-model row sorting, and stop fixtures encoding old severities A cap reachable under several request models emits a row each, so one can be exceeded while its siblings are fine. Pin that the exceeded row floats to the top and the siblings hold the server's order behind it rather than being reshuffled. Severity now turns on whether the row already carries the fact in a field, which is orthogonal to whether the row is dead, and all four combinations occur. Assert the truth table so no fixture can quietly reassert that severity implies deadness. --- .../templates/KeyBudgetsTableColumns.test.ts | 30 +++++++------- .../key_info_view.budgets_tab.test.tsx | 39 ++++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 20 ++++++---- 3 files changed, 67 insertions(+), 22 deletions(-) diff --git a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts index db2013d9e35..02106a29f7a 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts +++ b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts @@ -45,13 +45,18 @@ const THROTTLE_NOTE = { text: "this key opted into throttle_on_budget_exceeded, so exceeding it slows requests instead of blocking", } as const; -// Tagged info by the server, but it scopes which requests the row applies to rather than killing it. const END_USER_ROUTE_NOTE = { code: "end_user_route_only", - severity: "info", + severity: "warning", text: "only enforced on LLM routes that name this end user", } as const; +const COLD_MODEL_NOTE = { + code: "model_budget_fails_open", + severity: "info", + text: "no per-model counter exists yet; these budgets are cache-only and fail open until one does", +} as const; + const TEAM_MEMBER_AT_LIMIT: KeyBudgetEntry = { ...UNCONFIGURED_BUDGET, scope: "team_member", @@ -157,17 +162,16 @@ describe("cannotTrip", () => { expect(cannotTrip(WARNED)).toBe(false); }); - it("keeps an end_user row alive even though the server tags its route caveat as info", () => { - const endUser: KeyBudgetEntry = { ...HEALTHY, scope: "end_user", notes: [END_USER_ROUTE_NOTE] }; - expect(END_USER_ROUTE_NOTE.severity).toBe("info"); - expect(cannotTrip(endUser)).toBe(false); - }); - - it("ignores severity entirely, since dead codes ship under both values", () => { - expect(PROJECT_DEAD_NOTE.severity).toBe("warning"); - expect(cannotTrip(INERT)).toBe(true); - expect(ROLLING_NOTE.severity).toBe("info"); - expect(cannotTrip(WARNED)).toBe(false); + // Severity answers whether the row already carries the fact in a field, which is orthogonal to + // whether the row is dead. All four corners occur, so severity can never stand in for deadness. + it.each([ + ["info", "dead", COLD_MODEL_NOTE, true], + ["info", "live", ROLLING_NOTE, false], + ["warning", "dead", PROJECT_DEAD_NOTE, true], + ["warning", "live", END_USER_ROUTE_NOTE, false], + ])("treats a %s note that is %s by code, not by severity", (severity, _kind, note, dead) => { + expect(note.severity).toBe(severity); + expect(cannotTrip({ ...HEALTHY, notes: [note] })).toBe(dead); }); it("assumes a code this build predates is live, so an unknown caveat never hides a real blocker", () => { diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx index fc1c63a2dbb..595f31792d9 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx @@ -215,7 +215,7 @@ const WORST_CASE_NOTES = [ }, { code: "end_user_route_only", - severity: "info", + severity: "warning", text: "only enforced on LLM routes that name this end user", }, { @@ -469,6 +469,43 @@ describe("KeyInfoView Budgets tab", () => { expect(dataRows.filter((row) => row.textContent?.includes("bedrock/claude-opus-5"))).toHaveLength(1); }); + it("floats the one exceeded per-model row above its healthy siblings on the same cap", async () => { + const cap = { + ...UNCONFIGURED_BUDGET, + scope: "key_model", + entity_type: "key", + entity_label: "claude-opus-5", + max_budget: 40, + comparison: ">", + source: "key.model_max_budget[claude-opus-5]", + } as const; + const healthy: KeyBudgetEntry = { ...cap, entity_id: "claude-opus-5", spend: 1, remaining: 39, status: "ok" }; + const alsoHealthy: KeyBudgetEntry = { + ...cap, + entity_id: "vertex_ai/claude-opus-5", + spend: 2, + remaining: 38, + status: "ok", + }; + const over: KeyBudgetEntry = { + ...cap, + entity_id: "bedrock/claude-opus-5", + spend: 41, + remaining: -1, + status: "exceeded", + }; + mockBudgets([healthy, alsoHealthy, over]); + const panel = await renderAndOpenBudgetsTab(); + + const [, ...dataRows] = panel.getAllByRole("row"); + expect(dataRows[0]).toHaveTextContent("bedrock/claude-opus-5"); + expect(within(dataRows[0]).getByTestId("key-budget-blocking")).toBeInTheDocument(); + // Siblings keep the server's order behind it rather than being reshuffled among themselves. + expect(dataRows[1]).toHaveTextContent("claude-opus-5"); + expect(dataRows[2]).toHaveTextContent("vertex_ai/claude-opus-5"); + expect(panel.getAllByTestId("key-budget-blocking")).toHaveLength(1); + }); + it("renders notes in the order the server sent them, most specific to these numbers first", async () => { const endUser: KeyBudgetEntry = { ...UNCONFIGURED_BUDGET, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d8944633c64..6b5efd3b0ca 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -6847,9 +6847,10 @@ export interface paths { * - source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:` * - status: str - `unlimited`, `ok` or `exceeded` * - notes: list - Caveats worth knowing before trusting the row, each with a stable `code` - * to branch on, a `severity` of `info` or `warning` for codes a client does not know yet, - * and human-facing `text` that is free to be reworded. Ordered most to least specific to - * this row's numbers, and empty rather than null when there is nothing to say + * to branch on and human-facing `text` that is free to be reworded. `severity` is for a + * `code` a client does not know yet: `info` only explains a field the row already carries, + * `warning` carries the fact on its own. Ordered most to least specific to this row's + * numbers, and empty rather than null when there is nothing to say * * Example Curl: * ``` @@ -7544,9 +7545,10 @@ export interface paths { * - source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:` * - status: str - `unlimited`, `ok` or `exceeded` * - notes: list - Caveats worth knowing before trusting the row, each with a stable `code` - * to branch on, a `severity` of `info` or `warning` for codes a client does not know yet, - * and human-facing `text` that is free to be reworded. Ordered most to least specific to - * this row's numbers, and empty rather than null when there is nothing to say + * to branch on and human-facing `text` that is free to be reworded. `severity` is for a + * `code` a client does not know yet: `info` only explains a field the row already carries, + * `warning` carries the fact on its own. Ordered most to least specific to this row's + * numbers, and empty rather than null when there is nothing to say * * Example Curl: * ``` @@ -26402,8 +26404,10 @@ export interface components { * * ``code`` is the contract: map it to whatever treatment the caveat deserves. ``text`` is free to be * reworded and must not be matched on. ``severity`` exists for the code a client has not been taught - * yet, since this union grows: ``warning`` means the row's numbers may be incomplete or read as - * something they are not, and ``info`` means they are accurate and the note is only context. + * yet, since this union grows, and it turns on whether the row already carries the fact in a field: + * ``info`` means the note only explains something the row states anyway, like ``enforcement``, + * ``comparison`` or ``spend_state``, and ``warning`` means the note alone carries it, so the row + * cannot be taken at face value without reading it. */ KeyBudgetNote: { /**