fix(key budgets): classify note severity on whether the row carries the fact

`end_user_route_only` was tagged info while its text is the only place the row says
it applies to requests naming that end user. Nothing in `scope`, `max_budget` or
`comparison` scopes it, so a client that does not know the code had no way to learn
the row might not be in play, on the row most likely to answer "what blocked me".

The definition was the weaker half of that. "The numbers are accurate" left a note
about applicability unclassifiable, because a row can be perfectly accurate and still
not apply. Severity now turns on one question with no exceptions: does a field on the
row already carry this fact. `enforcement` carries alert_only, `comparison` carries
the reservation note, `window_start` carries the rolling window, `max_budget` carries
the personal budget a team key ignores, and `spend_state` carries the missing counter,
so those stay info. The five that only the note carries stay warning, and the end user
scoping note joins them. No other code changes, which is the reason to trust the rule.

A test now pins all eleven against the generated union, so a twelfth code fails until
someone decides which it is, mirroring the exhaustive switch the dashboard compiles.
This commit is contained in:
ryan-crabbe-berri 2026-08-19 18:18:36 -07:00
parent d16a51a969
commit 6a6809a16a
4 changed files with 45 additions and 9 deletions

View file

@ -139,8 +139,8 @@ _TAG_NOTE: Final = KeyBudgetNote(
)
_END_USER_ROUTE_NOTE: Final = KeyBudgetNote(
code="end_user_route_only",
severity="info",
text="only enforced on LLM routes that name this end user",
severity="warning",
text="applies only to requests that name this end user; nothing else on this row says so",
)
_CUSTOM_AUTH_END_USER_NOTE: Final = KeyBudgetNote(
code="custom_auth_may_override_end_user_cap",

View file

@ -3831,9 +3831,10 @@ async def key_budgets_fn(
- source: str - Where the limit is configured, e.g. `key.max_budget`, `budget_table:<id>`
- status: str - `unlimited`, `ok` or `exceeded`
- notes: list - Caveats worth knowing before trusting the row, each with a stable `code`
to branch on, a `severity` of `info` or `warning` for codes a client does not know yet,
and human-facing `text` that is free to be reworded. Ordered most to least specific to
this row's numbers, and empty rather than null when there is nothing to say
to branch on and human-facing `text` that is free to be reworded. `severity` is for a
`code` a client does not know yet: `info` only explains a field the row already carries,
`warning` carries the fact on its own. Ordered most to least specific to this row's
numbers, and empty rather than null when there is nothing to say
Example Curl:
```

View file

@ -157,8 +157,10 @@ class KeyBudgetNote(BaseModel):
``code`` is the contract: map it to whatever treatment the caveat deserves. ``text`` is free to be
reworded and must not be matched on. ``severity`` exists for the code a client has not been taught
yet, since this union grows: ``warning`` means the row's numbers may be incomplete or read as
something they are not, and ``info`` means they are accurate and the note is only context.
yet, since this union grows, and it turns on whether the row already carries the fact in a field:
``info`` means the note only explains something the row states anyway, like ``enforcement``,
``comparison`` or ``spend_state``, and ``warning`` means the note alone carries it, so the row
cannot be taken at face value without reading it.
"""
model_config = ConfigDict(frozen=True)

View file

@ -16472,10 +16472,13 @@ from litellm.proxy.management_endpoints.key_budget_resolver import ( # noqa: E4
resolve_key_budgets,
)
from litellm.types.proxy.management_endpoints.key_management_endpoints import ( # noqa: E402
BudgetNoteCode,
KeyBudgetEntry,
KeyBudgetNote,
)
from litellm.types.utils import BudgetConfig as _BudgetsBudgetConfig # noqa: E402
from types import MappingProxyType # noqa: E402
from typing import get_args as _budgets_get_args # noqa: E402
BudgetConfig = _BudgetsBudgetConfig
@ -17234,6 +17237,35 @@ async def test_key_budgets_call_an_unreadable_counter_unreadable_rather_than_rep
assert project_entry.spend_state == "live", "the recorded-spend scopes do not go through the counter"
def test_key_budgets_classify_every_note_code_and_leave_none_to_a_default():
"""
Severity is the only thing a client has for a code its build predates, so every code is classified
here on one rule: `info` when the row already carries the fact in a field, `warning` when the note
is the only place it appears. A twelfth code fails this until someone decides which it is.
"""
from litellm.proxy.management_endpoints import key_budget_resolver
notes = {
value.code: value for value in vars(key_budget_resolver).values() if isinstance(value, KeyBudgetNote)
}
assert set(notes) == set(_budgets_get_args(BudgetNoteCode)), "a code with no note, or a note with no code"
assert {code: note.severity for code, note in notes.items()} == {
# the row states it: `enforcement`, `comparison`, `window_start`, `max_budget`, `spend_state`
"alert_only": "info",
"reservation_blocks_at_limit": "info",
"rolling_window": "info",
"user_budget_not_applied_to_team_key": "info",
"model_budget_fails_open": "info",
# only the note states it
"custom_auth_may_override_end_user_cap": "warning",
"end_user_route_only": "warning",
"per_model_counters": "warning",
"project_spend_not_tracked": "warning",
"request_tags_add_budgets": "warning",
"throttled_instead_of_blocked": "warning",
}
@pytest.mark.asyncio
async def test_key_budgets_separate_caveats_that_rule_a_row_out_from_caveats_that_change_how_to_read_it():
"""Severity is what lets a reader skip a budget that cannot trip without skipping one that blocks early."""
@ -17243,8 +17275,9 @@ async def test_key_budgets_separate_caveats_that_rule_a_row_out_from_caveats_tha
assert severity_by_code["project_spend_not_tracked"] == "warning", "a budget that cannot trip is not a live row"
assert severity_by_code["request_tags_add_budgets"] == "warning", "the list of tag budgets is incomplete"
assert severity_by_code["reservation_blocks_at_limit"] == "info", "`comparison` already carries this"
assert severity_by_code["rolling_window"] == "info", "the numbers are right, the window just moves"
assert severity_by_code["alert_only"] == "info"
assert severity_by_code["rolling_window"] == "info", "`window_start` already carries this"
assert severity_by_code["alert_only"] == "info", "`enforcement` already carries this"
assert severity_by_code["end_user_route_only"] == "warning", "nothing else on the row scopes it"
@pytest.mark.asyncio