scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

test(ui): pin per-model row sorting, and stop fixtures encoding old severities

Browse source 
A cap reachable under several request models emits a row each, so one can be
exceeded while its siblings are fine. Pin that the exceeded row floats to the top
and the siblings hold the server's order behind it rather than being reshuffled.

Severity now turns on whether the row already carries the fact in a field, which
is orthogonal to whether the row is dead, and all four combinations occur. Assert
the truth table so no fixture can quietly reassert that severity implies deadness.
This commit is contained in:
ryan-crabbe-berri 2026-08-19 18:21:57 -07:00
parent 6a6809a16a
commit ed7cee5d50
3 changed files with 67 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
ui/litellm-dashboard/src/components/templates/KeyBudgetsTableColumns.test.ts
View file

@ -45,13 +45,18 @@ const THROTTLE_NOTE = {
text: "this key opted into throttle_on_budget_exceeded, so exceeding it slows requests instead of blocking",
} as const;
// Tagged info by the server, but it scopes which requests the row applies to rather than killing it.
const END_USER_ROUTE_NOTE = {
code: "end_user_route_only",
severity: "info",
severity: "warning",
text: "only enforced on LLM routes that name this end user",
} as const;
const COLD_MODEL_NOTE = {
code: "model_budget_fails_open",
severity: "info",
text: "no per-model counter exists yet; these budgets are cache-only and fail open until one does",
} as const;
const TEAM_MEMBER_AT_LIMIT: KeyBudgetEntry = {
...UNCONFIGURED_BUDGET,
scope: "team_member",
@ -157,17 +162,16 @@ describe("cannotTrip", () => {
expect(cannotTrip(WARNED)).toBe(false);
});
it("keeps an end_user row alive even though the server tags its route caveat as info", () => {
const endUser: KeyBudgetEntry = { ...HEALTHY, scope: "end_user", notes: [END_USER_ROUTE_NOTE] };
expect(END_USER_ROUTE_NOTE.severity).toBe("info");
expect(cannotTrip(endUser)).toBe(false);
});
it("ignores severity entirely, since dead codes ship under both values", () => {
expect(PROJECT_DEAD_NOTE.severity).toBe("warning");
expect(cannotTrip(INERT)).toBe(true);
expect(ROLLING_NOTE.severity).toBe("info");
expect(cannotTrip(WARNED)).toBe(false);
// Severity answers whether the row already carries the fact in a field, which is orthogonal to
// whether the row is dead. All four corners occur, so severity can never stand in for deadness.
it.each([
["info", "dead", COLD_MODEL_NOTE, true],
["info", "live", ROLLING_NOTE, false],
["warning", "dead", PROJECT_DEAD_NOTE, true],
["warning", "live", END_USER_ROUTE_NOTE, false],
])("treats a %s note that is %s by code, not by severity", (severity, _kind, note, dead) => {
expect(note.severity).toBe(severity);
expect(cannotTrip({ ...HEALTHY, notes: [note] })).toBe(dead);
});
it("assumes a code this build predates is live, so an unknown caveat never hides a real blocker", () => {

39
ui/litellm-dashboard/src/components/templates/key_info_view.budgets_tab.test.tsx
View file

@ -215,7 +215,7 @@ const WORST_CASE_NOTES = [
},
{
code: "end_user_route_only",
severity: "info",
severity: "warning",
text: "only enforced on LLM routes that name this end user",
},
{
@ -469,6 +469,43 @@ describe("KeyInfoView Budgets tab", () => {
expect(dataRows.filter((row) => row.textContent?.includes("bedrock/claude-opus-5"))).toHaveLength(1);
});
it("floats the one exceeded per-model row above its healthy siblings on the same cap", async () => {
const cap = {
...UNCONFIGURED_BUDGET,
scope: "key_model",
entity_type: "key",
entity_label: "claude-opus-5",
max_budget: 40,
comparison: ">",
source: "key.model_max_budget[claude-opus-5]",
} as const;
const healthy: KeyBudgetEntry = { ...cap, entity_id: "claude-opus-5", spend: 1, remaining: 39, status: "ok" };
const alsoHealthy: KeyBudgetEntry = {
...cap,
entity_id: "vertex_ai/claude-opus-5",
spend: 2,
remaining: 38,
status: "ok",
};
const over: KeyBudgetEntry = {
...cap,
entity_id: "bedrock/claude-opus-5",
spend: 41,
remaining: -1,
status: "exceeded",
};
mockBudgets([healthy, alsoHealthy, over]);
const panel = await renderAndOpenBudgetsTab();
const [, ...dataRows] = panel.getAllByRole("row");
expect(dataRows[0]).toHaveTextContent("bedrock/claude-opus-5");
expect(within(dataRows[0]).getByTestId("key-budget-blocking")).toBeInTheDocument();
// Siblings keep the server's order behind it rather than being reshuffled among themselves.
expect(dataRows[1]).toHaveTextContent("claude-opus-5");
expect(dataRows[2]).toHaveTextContent("vertex_ai/claude-opus-5");
expect(panel.getAllByTestId("key-budget-blocking")).toHaveLength(1);
});
it("renders notes in the order the server sent them, most specific to these numbers first", async () => {
const endUser: KeyBudgetEntry = {
...UNCONFIGURED_BUDGET,

20
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -6847,9 +6847,10 @@ export interface paths {
* - source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:<id>`
* - status: str - `unlimited`, `ok` or `exceeded`
* - notes: list - Caveats worth knowing before trusting the row, each with a stable `code`
* to branch on, a `severity` of `info` or `warning` for codes a client does not know yet,
* and human-facing `text` that is free to be reworded. Ordered most to least specific to
* this row's numbers, and empty rather than null when there is nothing to say
* to branch on and human-facing `text` that is free to be reworded. `severity` is for a
* `code` a client does not know yet: `info` only explains a field the row already carries,
* `warning` carries the fact on its own. Ordered most to least specific to this row's
* numbers, and empty rather than null when there is nothing to say
*
* Example Curl:
* ```
@ -7544,9 +7545,10 @@ export interface paths {
* - source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:<id>`
* - status: str - `unlimited`, `ok` or `exceeded`
* - notes: list - Caveats worth knowing before trusting the row, each with a stable `code`
* to branch on, a `severity` of `info` or `warning` for codes a client does not know yet,
* and human-facing `text` that is free to be reworded. Ordered most to least specific to
* this row's numbers, and empty rather than null when there is nothing to say
* to branch on and human-facing `text` that is free to be reworded. `severity` is for a
* `code` a client does not know yet: `info` only explains a field the row already carries,
* `warning` carries the fact on its own. Ordered most to least specific to this row's
* numbers, and empty rather than null when there is nothing to say
*
* Example Curl:
* ```
@ -26402,8 +26404,10 @@ export interface components {
*
* ``code`` is the contract: map it to whatever treatment the caveat deserves. ``text`` is free to be
* reworded and must not be matched on. ``severity`` exists for the code a client has not been taught
* yet, since this union grows: ``warning`` means the row's numbers may be incomplete or read as
* something they are not, and ``info`` means they are accurate and the note is only context.
* yet, since this union grows, and it turns on whether the row already carries the fact in a field:
* ``info`` means the note only explains something the row states anyway, like ``enforcement``,
* ``comparison`` or ``spend_state``, and ``warning`` means the note alone carries it, so the row
* cannot be taken at face value without reading it.
*/
KeyBudgetNote: {
/**