From d0bf848d912a785dc1bf29901908122c5d282ea0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sun, 15 Nov 2026 16:49:47 -0800 Subject: [PATCH] fix(dynamic_rate_limiter): charge tokens used while the model was idle against a team's class share (#51952) --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 23 ++-- ...rity_class_share_light_team_not_starved.py | 107 ++++++++++++++++++ .../hooks/test_dynamic_rate_limiter_v3.py | 29 +++++ 3 files changed, 143 insertions(+), 16 deletions(-) create mode 100644 tests/integration/routing/test_priority_class_share_light_team_not_starved.py diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 0a7a6685e52..02f79a7897e 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -1374,16 +1374,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): llm_provider=llm_provider, ) - # If priority is NOT enforced (saturation below threshold) but - # priority_descriptors exist, increment them for tracking only — no - # check, no rollback. This matches the prior tracking semantics. - # - # Using the non-atomic should_rate_limit (instead of - # atomic_check_and_increment_by_n) is intentional here: we don't want - # to enforce the limit, we only want to bump the counter so the - # priority allocation has accurate usage when it later becomes - # enforced. The increment-then-check semantics of should_rate_limit - # are fine because we ignore the OVER_LIMIT response. + # Below saturation the priority counters are tracked, not enforced, so an + # OVER_LIMIT here is ignored. The atomic call adds one request and no + # tokens, real usage lands on the token counter at settlement. for kind in BORROWER_KINDS: state = request_stash.borrowing_shares[kind] reservation = reservations.get(kind, 0) @@ -1402,13 +1395,11 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ) if tightest_held is not None: self._cap_admitted_output(tightest_held[1], tightest_held[2], data, call_type) - if priority_descriptors and share is not None and not should_enforce_priority: - get_or_create_request_stash().rate_limit_response = atomic_response - elif priority_descriptors and not should_enforce_priority: - priority_tracking_response: Final = await self.v3_limiter.should_rate_limit( - descriptors=priority_descriptors, + if priority_descriptors and not should_enforce_priority: + priority_tracking_response: Final = await self.v3_limiter.atomic_check_and_increment_by_n( + descriptors=list(priority_descriptors), + increments=[per_request_increment for _ in priority_descriptors], parent_otel_span=user_api_key_dict.parent_otel_span, - read_only=False, ) get_or_create_request_stash().rate_limit_response = RateLimitResponse( overall_code=atomic_response["overall_code"], diff --git a/tests/integration/routing/test_priority_class_share_light_team_not_starved.py b/tests/integration/routing/test_priority_class_share_light_team_not_starved.py new file mode 100644 index 00000000000..a2bc8464140 --- /dev/null +++ b/tests/integration/routing/test_priority_class_share_light_team_not_starved.py @@ -0,0 +1,107 @@ +import json +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +OPENAI_MODEL: Final = "gpt-4.1-mini" +MODEL_TPM: Final = 1000 +TEAM_SHARE: Final = MODEL_TPM // 2 +TOKENS_PER_CALL: Final = 30 +UPSTREAM_REPLY: Final = json.dumps( + { + "id": "chatcmpl_priority_class_share_borrowed", + "object": "chat.completion", + "created": 1700000000, + "model": OPENAI_MODEL, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "shared"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": TOKENS_PER_CALL}, + } +).encode() + + +def _respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/chat/completions", request.target + return Reply(body=UPSTREAM_REPLY) + + +@pytest.fixture(scope="module") +def model_saturating_at_forty_percent(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, str]]: + directory: Final = tmp_path_factory.mktemp("priority-class-share-borrowed") + configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration["litellm_settings"] = { + **configuration["litellm_settings"], + "callbacks": ["dynamic_rate_limiter_v3"], + "priority_reservation": {"interactive": 0.6, "batch": 0.4}, + "priority_reservation_settings": { + "default_priority": 1.0, + "saturation_threshold": 0.4, + "saturation_check_cache_ttl": 0, + "priority_class_share_config": {"split_equally_among_active": ["team"]}, + }, + } + path: Final = directory / "priority_class_share_borrowed.yaml" + path.write_text(yaml.safe_dump(configuration)) + with ( + gateway_from_environment() as rig, + wire_server(_respond) as wire, + owned_proxy(rig, directory, {}, config=path) as owned, + ): + yield owned, wire.url + + +def _team_key(scenario: Scenario, priority: str) -> str: + return scenario.key(team_id=scenario.team(metadata={"priority": priority})) + + +def _call(owned: Gateway, model: str, key: str) -> httpx.Response: + return owned.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"class share {uuid.uuid4().hex}"}]}, + key=key, + ) + + +def _spend_until_blocked(owned: Gateway, model: str, key: str, max_calls: int) -> tuple[httpx.Response, ...]: + def calls() -> Iterator[httpx.Response]: + for _ in range(max_calls): + response: Final = _call(owned, model, key) + yield response + if response.status_code == 429: + return + + return tuple(calls()) + + +def test_team_that_borrowed_before_saturation_cannot_starve_a_teammate_within_its_share( + model_saturating_at_forty_percent: tuple[Gateway, str], +) -> None: + """Two interactive teams, batch idle, each owed half the model. The greedy one sends from an idle model until it + is refused, and the light one must still be served up to its own half.""" + owned, upstream = model_saturating_at_forty_percent + with owned.scenario() as scenario: + model: Final = scenario.model( + model=f"openai/{OPENAI_MODEL}", api_base=f"{upstream}/v1", api_key="synthetic-openai-key", tpm=MODEL_TPM + ) + greedy: Final = _team_key(scenario, "interactive") + light: Final = _team_key(scenario, "interactive") + assert _call(owned, model, light).status_code == 200 + + greedy_responses: Final = _spend_until_blocked(owned, model, greedy, max_calls=MODEL_TPM // TOKENS_PER_CALL + 1) + blocked: Final = greedy_responses[-1] + assert blocked.status_code == 429, blocked.text + assert blocked.headers.get("rate_limited_by") == "priority_class_share_saturated", blocked.text + greedy_served: Final = (len(greedy_responses) - 1) * TOKENS_PER_CALL + assert TEAM_SHARE <= greedy_served < TEAM_SHARE + 2 * TOKENS_PER_CALL, greedy_served + + light_responses: Final = _spend_until_blocked(owned, model, light, max_calls=MODEL_TPM // TOKENS_PER_CALL + 1) + light_served: Final = (1 + sum(1 for r in light_responses if r.status_code == 200)) * TOKENS_PER_CALL + assert light_served >= TEAM_SHARE - 2 * TOKENS_PER_CALL, (light_served, light_responses[-1].text) diff --git a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py index 0c431ae9c38..95e75ba6774 100644 --- a/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -2739,6 +2739,35 @@ async def test_two_interactive_teams_each_keep_half_of_the_interactive_share(mon await _check(handler, _class_team("team-a"), saturation=0.52, priority="interactive") +@pytest.mark.asyncio +async def test_tokens_a_team_used_while_the_model_was_idle_count_against_its_class_share( + monkeypatch: pytest.MonkeyPatch, +): + handler: Final = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-a"), "interactive", 20, saturation=0.0) + await _burn(handler, _class_team("team-b"), "interactive", 400, saturation=0.02) + await _burn(handler, _class_team("team-b"), "interactive", 100, saturation=0.42) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-b"), saturation=0.52, priority="interactive") + _held_to_class_share(exc.value) + assert "Share: 500, Remaining: 0, Model saturation: 52.0%" in str(exc.value.detail) + await _burn(handler, _class_team("team-a"), "interactive", 470, saturation=0.52) + + +@pytest.mark.asyncio +async def test_requests_sent_while_the_model_was_idle_charge_only_their_tokens_to_the_class_share( + monkeypatch: pytest.MonkeyPatch, +): + handler: Final = _borrowing_handler(monkeypatch, borrows=False, splits=True) + await _burn(handler, _class_team("team-a"), "interactive", 20, saturation=0.0) + for _ in range(5): + await _burn(handler, _class_team("team-b"), "interactive", 99, saturation=0.02) + await _burn(handler, _class_team("team-b"), "interactive", 5, saturation=0.52) + with pytest.raises(ProxyRateLimitError) as exc: + await _check(handler, _class_team("team-b"), saturation=0.52, priority="interactive") + _held_to_class_share(exc.value) + + @pytest.mark.asyncio async def test_the_only_active_class_gets_the_whole_model_instead_of_its_weight(monkeypatch: pytest.MonkeyPatch): handler = _borrowing_handler(monkeypatch, borrows=False, splits=True)