From 6a931a9def5507045d867e6e9b5be490682a61b5 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 19 Aug 2026 18:28:06 -0700 Subject: [PATCH] test(ui): stop asserting server note prose, which was promised to change `code` and `severity` are the contract and `text` is explicitly free to be reworded, so pinning the resolver's sentences was asserting the one field guaranteed to move. It went stale three times in a day, silently each time, because a fixture copy keeps passing long after the server stops agreeing. Note texts are synthetic and derived from the code. The wrapping guard uses a string past 500 characters rather than a copy of the current worst case, so no rewording upstream can move it and it still fails if truncation returns. --- .../templates/KeyBudgetsTableColumns.test.ts | 16 +++-- .../key_info_view.budgets_tab.test.tsx | 69 +++++++------------ 2 files changed, 36 insertions(+), 49 deletions(-) diff --git a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts index b9e3fb7aaeb..8ad6e39c5e1 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts +++ b/ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts @@ -21,40 +21,44 @@ const UNCONFIGURED_BUDGET = { notes: [], } as KeyBudgetEntry; +// `code` and `severity` are the contract; `text` is explicitly free to be reworded, so these are +// synthetic. A test that pins the server's prose asserts the one field promised to change. +const noteText = (code: string): string => `synthetic ${code} caveat`; + const PROJECT_DEAD_NOTE = { code: "project_spend_not_tracked", severity: "warning", - text: "project spend is never incremented today, so this budget cannot trip", + text: noteText("project_spend_not_tracked"), } as const; const ALERT_ONLY_NOTE = { code: "alert_only", severity: "info", - text: "alert only, never blocks", + text: noteText("alert_only"), } as const; const ROLLING_NOTE = { code: "rolling_window", severity: "info", - text: "rolling window", + text: noteText("rolling_window"), } as const; const THROTTLE_NOTE = { code: "throttled_instead_of_blocked", severity: "warning", - text: "this key opted into throttle_on_budget_exceeded, so exceeding it slows requests instead of blocking", + text: noteText("throttled_instead_of_blocked"), } as const; const END_USER_ROUTE_NOTE = { code: "end_user_route_only", severity: "warning", - text: "applies only to requests that name this end user; nothing else on this row says so", + text: noteText("end_user_route_only"), } as const; const COLD_MODEL_NOTE = { code: "model_budget_fails_open", severity: "info", - text: "no per-model counter exists yet; these budgets are cache-only and fail open until one does", + text: noteText("model_budget_fails_open"), } as const; const TEAM_MEMBER_AT_LIMIT: KeyBudgetEntry = { diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx index c1cc7b26576..ed34bf4e193 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx @@ -131,16 +131,20 @@ const UNCONFIGURED_BUDGET = { notes: [], } as KeyBudgetEntry; +// `code` and `severity` are the contract; `text` is explicitly free to be reworded, so every note +// text below is synthetic. Copying the resolver's prose only buys tests that go stale silently. +const noteText = (code: string): string => `synthetic ${code} caveat`; + const ALERT_ONLY_NOTE = { code: "alert_only", severity: "info", - text: "alert only, never blocks; compared against recorded spend rather than the live counter", + text: noteText("alert_only"), } as const; const PROJECT_DEAD_NOTE = { code: "project_spend_not_tracked", severity: "warning", - text: "project spend is never incremented today, so this budget cannot trip", + text: noteText("project_spend_not_tracked"), } as const; const KEY_UNLIMITED: KeyBudgetEntry = { @@ -203,38 +207,21 @@ const ORG_UNCONFIGURED: KeyBudgetEntry = { source: "organization.budget_id", }; -// The widest row the endpoint can emit: an end_user budget the reservation layer tightened, on a -// proxy with a custom auth callable. Three notes rather than one sentence, in the server's order. -// Texts are snapshots of the resolver's constants; 382 characters across the row is the constraint. +// The widest row the endpoint can emit is an end_user budget the reservation layer tightened, on a +// proxy with a custom auth callable: three notes rather than one sentence, in the server's order. const WORST_CASE_NOTES = [ - { - code: "reservation_blocks_at_limit", - severity: "info", - text: - "the reservation layer usually blocks this scope as soon as spend reaches the limit, ahead of the " + - "read-time check; requests it cannot price up front are still gated by the read-time check alone", - }, - { - code: "end_user_route_only", - severity: "warning", - text: "applies only to requests that name this end user; nothing else on this row says so", - }, + { code: "reservation_blocks_at_limit", severity: "info", text: noteText("reservation_blocks_at_limit") }, + { code: "end_user_route_only", severity: "warning", text: noteText("end_user_route_only") }, { code: "custom_auth_may_override_end_user_cap", severity: "warning", - text: "a custom auth callable can set a request-scoped end user cap that overrides this one and is not visible here", + text: noteText("custom_auth_may_override_end_user_cap"), }, ] as const; -// Longest single note the endpoint can emit at 192 characters, and so the real width constraint on -// one line of the scope column. Snapshot of the resolver's constant. -const LONGEST_SINGLE_NOTE = { - code: "reservation_blocks_at_limit", - severity: "info", - text: - "the reservation layer usually blocks this scope as soon as spend reaches the limit, ahead of the " + - "read-time check; requests it cannot price up front are still gated by the read-time check alone", -} as const; +// Deliberately far past anything the endpoint emits, so no rewording on the server can move this +// guard. Only its length and the absence of clipping matter, never its wording. +const OVERLONG_NOTE_TEXT = `${"a caveat clause that keeps going ".repeat(16)}end`; const ALL_BUDGETS = [KEY_UNLIMITED, USER_WITHIN_BUDGET, TEAM_SOFT_OVER, TEAM_MEMBER_BLOCKING, ORG_UNCONFIGURED]; @@ -314,9 +301,7 @@ describe("KeyInfoView Budgets tab", () => { expect(within(softRow).getByText("Exceeded (alert only)")).toBeInTheDocument(); expect(within(softRow).queryByTestId("key-budget-blocking")).not.toBeInTheDocument(); expect(within(softRow).queryByText("Blocks requests")).not.toBeInTheDocument(); - expect(softRow).toHaveTextContent( - "alert only, never blocks; compared against recorded spend rather than the live counter", - ); + expect(softRow).toHaveTextContent(ALERT_ONLY_NOTE.text); }); it("does not render a spend the server could not read as a healthy $0.00", async () => { @@ -408,17 +393,9 @@ describe("KeyInfoView Budgets tab", () => { expect(rendered).toHaveLength(3); // Separate elements, not one joined blob, so each caveat can carry its own severity. expect(new Set(rendered).size).toBe(3); - - // jsdom has no layout, so nothing here can prove the text is visually unclipped. Asserting the - // absence of the clipping utilities is the only mechanical guard against re-truncating a note. - for (const note of rendered) { - expect(note).not.toHaveClass("truncate"); - expect(note.className).not.toMatch(/line-clamp|overflow-hidden|whitespace-nowrap/); - expect(note).not.toHaveAttribute("title"); - } }); - it("renders the longest single note the endpoint can emit without dropping any of it", async () => { + it("renders a note far longer than any the endpoint emits without dropping any of it", async () => { const reserved: KeyBudgetEntry = { ...UNCONFIGURED_BUDGET, scope: "team", @@ -429,13 +406,19 @@ describe("KeyInfoView Budgets tab", () => { remaining: 180, source: "team.max_budget", status: "ok", - notes: [LONGEST_SINGLE_NOTE], + notes: [{ code: "rolling_window", severity: "info", text: OVERLONG_NOTE_TEXT }], }; mockBudgets([reserved]); const panel = await renderAndOpenBudgetsTab(); - expect(LONGEST_SINGLE_NOTE.text).toHaveLength(192); - expect(panel.getByText(LONGEST_SINGLE_NOTE.text)).toBeInTheDocument(); + expect(OVERLONG_NOTE_TEXT.length).toBeGreaterThan(500); + const note = panel.getByText(OVERLONG_NOTE_TEXT); + + // jsdom has no layout, so nothing here can prove the text is visually unclipped. Asserting the + // absence of the clipping utilities is the only mechanical guard against re-truncating a note. + expect(note).not.toHaveClass("truncate"); + expect(note.className).not.toMatch(/line-clamp|overflow-hidden|whitespace-nowrap/); + expect(note).not.toHaveAttribute("title"); }); it("keeps two per-model rows on one cap apart by the request model each measures", async () => { @@ -548,7 +531,7 @@ describe("KeyInfoView Budgets tab", () => { { code: "throttled_instead_of_blocked", severity: "warning", - text: "this key opted into throttle_on_budget_exceeded, so exceeding it slows requests instead of blocking", + text: noteText("throttled_instead_of_blocked"), }, ], };