fix(dynamic_rate_limiter): charge tokens used while the model was idle against a team's class share (#51952)

This commit is contained in:
Kerry Lu 2026-11-15 16:49:47 -08:00 • committed by GitHub
parent 2aa57d7e04
commit d0bf848d91
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 143 additions and 16 deletions

View file

@ -1374,16 +1374,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
llm_provider=llm_provider,
)
# If priority is NOT enforced (saturation below threshold) but
# priority_descriptors exist, increment them for tracking only — no
# check, no rollback. This matches the prior tracking semantics.
#
# Using the non-atomic should_rate_limit (instead of
# atomic_check_and_increment_by_n) is intentional here: we don't want
# to enforce the limit, we only want to bump the counter so the
# priority allocation has accurate usage when it later becomes
# enforced. The increment-then-check semantics of should_rate_limit
# are fine because we ignore the OVER_LIMIT response.
# Below saturation the priority counters are tracked, not enforced, so an
# OVER_LIMIT here is ignored. The atomic call adds one request and no
# tokens, real usage lands on the token counter at settlement.
for kind in BORROWER_KINDS:
state = request_stash.borrowing_shares[kind]
reservation = reservations.get(kind, 0)
@ -1402,13 +1395,11 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
)
if tightest_held is not None:
self._cap_admitted_output(tightest_held[1], tightest_held[2], data, call_type)
if priority_descriptors and share is not None and not should_enforce_priority:
get_or_create_request_stash().rate_limit_response = atomic_response
elif priority_descriptors and not should_enforce_priority:
priority_tracking_response: Final = await self.v3_limiter.should_rate_limit(
descriptors=priority_descriptors,
if priority_descriptors and not should_enforce_priority:
priority_tracking_response: Final = await self.v3_limiter.atomic_check_and_increment_by_n(
descriptors=list(priority_descriptors),
increments=[per_request_increment for _ in priority_descriptors],
parent_otel_span=user_api_key_dict.parent_otel_span,
read_only=False,
)
get_or_create_request_stash().rate_limit_response = RateLimitResponse(
overall_code=atomic_response["overall_code"],

View file

@ -0,0 +1,107 @@
import json
import uuid
from collections.abc import Iterator
from pathlib import Path
from typing import Final
import httpx
import pytest
import yaml
from integration._support.client import Gateway, Scenario, gateway_from_environment
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, wire_server
OPENAI_MODEL: Final = "gpt-4.1-mini"
MODEL_TPM: Final = 1000
TEAM_SHARE: Final = MODEL_TPM // 2
TOKENS_PER_CALL: Final = 30
UPSTREAM_REPLY: Final = json.dumps(
{
"id": "chatcmpl_priority_class_share_borrowed",
"object": "chat.completion",
"created": 1700000000,
"model": OPENAI_MODEL,
"choices": [{"index": 0, "message": {"role": "assistant", "content": "shared"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": TOKENS_PER_CALL},
}
).encode()
def _respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/v1/chat/completions", request.target
return Reply(body=UPSTREAM_REPLY)
@pytest.fixture(scope="module")
def model_saturating_at_forty_percent(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, str]]:
directory: Final = tmp_path_factory.mktemp("priority-class-share-borrowed")
configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
configuration["litellm_settings"] = {
**configuration["litellm_settings"],
"callbacks": ["dynamic_rate_limiter_v3"],
"priority_reservation": {"interactive": 0.6, "batch": 0.4},
"priority_reservation_settings": {
"default_priority": 1.0,
"saturation_threshold": 0.4,
"saturation_check_cache_ttl": 0,
"priority_class_share_config": {"split_equally_among_active": ["team"]},
},
}
path: Final = directory / "priority_class_share_borrowed.yaml"
path.write_text(yaml.safe_dump(configuration))
with (
gateway_from_environment() as rig,
wire_server(_respond) as wire,
owned_proxy(rig, directory, {}, config=path) as owned,
):
yield owned, wire.url
def _team_key(scenario: Scenario, priority: str) -> str:
return scenario.key(team_id=scenario.team(metadata={"priority": priority}))
def _call(owned: Gateway, model: str, key: str) -> httpx.Response:
return owned.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": f"class share {uuid.uuid4().hex}"}]},
key=key,
)
def _spend_until_blocked(owned: Gateway, model: str, key: str, max_calls: int) -> tuple[httpx.Response, ...]:
def calls() -> Iterator[httpx.Response]:
for _ in range(max_calls):
response: Final = _call(owned, model, key)
yield response
if response.status_code == 429:
return
return tuple(calls())
def test_team_that_borrowed_before_saturation_cannot_starve_a_teammate_within_its_share(
model_saturating_at_forty_percent: tuple[Gateway, str],
) -> None:
"""Two interactive teams, batch idle, each owed half the model. The greedy one sends from an idle model until it
is refused, and the light one must still be served up to its own half."""
owned, upstream = model_saturating_at_forty_percent
with owned.scenario() as scenario:
model: Final = scenario.model(
model=f"openai/{OPENAI_MODEL}", api_base=f"{upstream}/v1", api_key="synthetic-openai-key", tpm=MODEL_TPM
)
greedy: Final = _team_key(scenario, "interactive")
light: Final = _team_key(scenario, "interactive")
assert _call(owned, model, light).status_code == 200
greedy_responses: Final = _spend_until_blocked(owned, model, greedy, max_calls=MODEL_TPM // TOKENS_PER_CALL + 1)
blocked: Final = greedy_responses[-1]
assert blocked.status_code == 429, blocked.text
assert blocked.headers.get("rate_limited_by") == "priority_class_share_saturated", blocked.text
greedy_served: Final = (len(greedy_responses) - 1) * TOKENS_PER_CALL
assert TEAM_SHARE <= greedy_served < TEAM_SHARE + 2 * TOKENS_PER_CALL, greedy_served
light_responses: Final = _spend_until_blocked(owned, model, light, max_calls=MODEL_TPM // TOKENS_PER_CALL + 1)
light_served: Final = (1 + sum(1 for r in light_responses if r.status_code == 200)) * TOKENS_PER_CALL
assert light_served >= TEAM_SHARE - 2 * TOKENS_PER_CALL, (light_served, light_responses[-1].text)

View file

@ -2739,6 +2739,35 @@ async def test_two_interactive_teams_each_keep_half_of_the_interactive_share(mon
await _check(handler, _class_team("team-a"), saturation=0.52, priority="interactive")
@pytest.mark.asyncio
async def test_tokens_a_team_used_while_the_model_was_idle_count_against_its_class_share(
monkeypatch: pytest.MonkeyPatch,
):
handler: Final = _borrowing_handler(monkeypatch, borrows=False, splits=True)
await _burn(handler, _class_team("team-a"), "interactive", 20, saturation=0.0)
await _burn(handler, _class_team("team-b"), "interactive", 400, saturation=0.02)
await _burn(handler, _class_team("team-b"), "interactive", 100, saturation=0.42)
with pytest.raises(ProxyRateLimitError) as exc:
await _check(handler, _class_team("team-b"), saturation=0.52, priority="interactive")
_held_to_class_share(exc.value)
assert "Share: 500, Remaining: 0, Model saturation: 52.0%" in str(exc.value.detail)
await _burn(handler, _class_team("team-a"), "interactive", 470, saturation=0.52)
@pytest.mark.asyncio
async def test_requests_sent_while_the_model_was_idle_charge_only_their_tokens_to_the_class_share(
monkeypatch: pytest.MonkeyPatch,
):
handler: Final = _borrowing_handler(monkeypatch, borrows=False, splits=True)
await _burn(handler, _class_team("team-a"), "interactive", 20, saturation=0.0)
for _ in range(5):
await _burn(handler, _class_team("team-b"), "interactive", 99, saturation=0.02)
await _burn(handler, _class_team("team-b"), "interactive", 5, saturation=0.52)
with pytest.raises(ProxyRateLimitError) as exc:
await _check(handler, _class_team("team-b"), saturation=0.52, priority="interactive")
_held_to_class_share(exc.value)
@pytest.mark.asyncio
async def test_the_only_active_class_gets_the_whole_model_instead_of_its_weight(monkeypatch: pytest.MonkeyPatch):
handler = _borrowing_handler(monkeypatch, borrows=False, splits=True)