mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(dynamic_rate_limiter): charge tokens used while the model was idle against a team's class share (#51952)
This commit is contained in:
parent
2aa57d7e04
commit
d0bf848d91
3 changed files with 143 additions and 16 deletions
|
|
@ -1374,16 +1374,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
|
|||
llm_provider=llm_provider,
|
||||
)
|
||||
|
||||
# If priority is NOT enforced (saturation below threshold) but
|
||||
# priority_descriptors exist, increment them for tracking only — no
|
||||
# check, no rollback. This matches the prior tracking semantics.
|
||||
#
|
||||
# Using the non-atomic should_rate_limit (instead of
|
||||
# atomic_check_and_increment_by_n) is intentional here: we don't want
|
||||
# to enforce the limit, we only want to bump the counter so the
|
||||
# priority allocation has accurate usage when it later becomes
|
||||
# enforced. The increment-then-check semantics of should_rate_limit
|
||||
# are fine because we ignore the OVER_LIMIT response.
|
||||
# Below saturation the priority counters are tracked, not enforced, so an
|
||||
# OVER_LIMIT here is ignored. The atomic call adds one request and no
|
||||
# tokens, real usage lands on the token counter at settlement.
|
||||
for kind in BORROWER_KINDS:
|
||||
state = request_stash.borrowing_shares[kind]
|
||||
reservation = reservations.get(kind, 0)
|
||||
|
|
@ -1402,13 +1395,11 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
|
|||
)
|
||||
if tightest_held is not None:
|
||||
self._cap_admitted_output(tightest_held[1], tightest_held[2], data, call_type)
|
||||
if priority_descriptors and share is not None and not should_enforce_priority:
|
||||
get_or_create_request_stash().rate_limit_response = atomic_response
|
||||
elif priority_descriptors and not should_enforce_priority:
|
||||
priority_tracking_response: Final = await self.v3_limiter.should_rate_limit(
|
||||
descriptors=priority_descriptors,
|
||||
if priority_descriptors and not should_enforce_priority:
|
||||
priority_tracking_response: Final = await self.v3_limiter.atomic_check_and_increment_by_n(
|
||||
descriptors=list(priority_descriptors),
|
||||
increments=[per_request_increment for _ in priority_descriptors],
|
||||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
read_only=False,
|
||||
)
|
||||
get_or_create_request_stash().rate_limit_response = RateLimitResponse(
|
||||
overall_code=atomic_response["overall_code"],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,107 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, Scenario, gateway_from_environment
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
OPENAI_MODEL: Final = "gpt-4.1-mini"
|
||||
MODEL_TPM: Final = 1000
|
||||
TEAM_SHARE: Final = MODEL_TPM // 2
|
||||
TOKENS_PER_CALL: Final = 30
|
||||
UPSTREAM_REPLY: Final = json.dumps(
|
||||
{
|
||||
"id": "chatcmpl_priority_class_share_borrowed",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": OPENAI_MODEL,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "shared"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": TOKENS_PER_CALL},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _respond(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/v1/chat/completions", request.target
|
||||
return Reply(body=UPSTREAM_REPLY)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def model_saturating_at_forty_percent(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, str]]:
|
||||
directory: Final = tmp_path_factory.mktemp("priority-class-share-borrowed")
|
||||
configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
configuration["litellm_settings"] = {
|
||||
**configuration["litellm_settings"],
|
||||
"callbacks": ["dynamic_rate_limiter_v3"],
|
||||
"priority_reservation": {"interactive": 0.6, "batch": 0.4},
|
||||
"priority_reservation_settings": {
|
||||
"default_priority": 1.0,
|
||||
"saturation_threshold": 0.4,
|
||||
"saturation_check_cache_ttl": 0,
|
||||
"priority_class_share_config": {"split_equally_among_active": ["team"]},
|
||||
},
|
||||
}
|
||||
path: Final = directory / "priority_class_share_borrowed.yaml"
|
||||
path.write_text(yaml.safe_dump(configuration))
|
||||
with (
|
||||
gateway_from_environment() as rig,
|
||||
wire_server(_respond) as wire,
|
||||
owned_proxy(rig, directory, {}, config=path) as owned,
|
||||
):
|
||||
yield owned, wire.url
|
||||
|
||||
|
||||
def _team_key(scenario: Scenario, priority: str) -> str:
|
||||
return scenario.key(team_id=scenario.team(metadata={"priority": priority}))
|
||||
|
||||
|
||||
def _call(owned: Gateway, model: str, key: str) -> httpx.Response:
|
||||
return owned.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": f"class share {uuid.uuid4().hex}"}]},
|
||||
key=key,
|
||||
)
|
||||
|
||||
|
||||
def _spend_until_blocked(owned: Gateway, model: str, key: str, max_calls: int) -> tuple[httpx.Response, ...]:
|
||||
def calls() -> Iterator[httpx.Response]:
|
||||
for _ in range(max_calls):
|
||||
response: Final = _call(owned, model, key)
|
||||
yield response
|
||||
if response.status_code == 429:
|
||||
return
|
||||
|
||||
return tuple(calls())
|
||||
|
||||
|
||||
def test_team_that_borrowed_before_saturation_cannot_starve_a_teammate_within_its_share(
|
||||
model_saturating_at_forty_percent: tuple[Gateway, str],
|
||||
) -> None:
|
||||
"""Two interactive teams, batch idle, each owed half the model. The greedy one sends from an idle model until it
|
||||
is refused, and the light one must still be served up to its own half."""
|
||||
owned, upstream = model_saturating_at_forty_percent
|
||||
with owned.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"openai/{OPENAI_MODEL}", api_base=f"{upstream}/v1", api_key="synthetic-openai-key", tpm=MODEL_TPM
|
||||
)
|
||||
greedy: Final = _team_key(scenario, "interactive")
|
||||
light: Final = _team_key(scenario, "interactive")
|
||||
assert _call(owned, model, light).status_code == 200
|
||||
|
||||
greedy_responses: Final = _spend_until_blocked(owned, model, greedy, max_calls=MODEL_TPM // TOKENS_PER_CALL + 1)
|
||||
blocked: Final = greedy_responses[-1]
|
||||
assert blocked.status_code == 429, blocked.text
|
||||
assert blocked.headers.get("rate_limited_by") == "priority_class_share_saturated", blocked.text
|
||||
greedy_served: Final = (len(greedy_responses) - 1) * TOKENS_PER_CALL
|
||||
assert TEAM_SHARE <= greedy_served < TEAM_SHARE + 2 * TOKENS_PER_CALL, greedy_served
|
||||
|
||||
light_responses: Final = _spend_until_blocked(owned, model, light, max_calls=MODEL_TPM // TOKENS_PER_CALL + 1)
|
||||
light_served: Final = (1 + sum(1 for r in light_responses if r.status_code == 200)) * TOKENS_PER_CALL
|
||||
assert light_served >= TEAM_SHARE - 2 * TOKENS_PER_CALL, (light_served, light_responses[-1].text)
|
||||
|
|
@ -2739,6 +2739,35 @@ async def test_two_interactive_teams_each_keep_half_of_the_interactive_share(mon
|
|||
await _check(handler, _class_team("team-a"), saturation=0.52, priority="interactive")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokens_a_team_used_while_the_model_was_idle_count_against_its_class_share(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
handler: Final = _borrowing_handler(monkeypatch, borrows=False, splits=True)
|
||||
await _burn(handler, _class_team("team-a"), "interactive", 20, saturation=0.0)
|
||||
await _burn(handler, _class_team("team-b"), "interactive", 400, saturation=0.02)
|
||||
await _burn(handler, _class_team("team-b"), "interactive", 100, saturation=0.42)
|
||||
with pytest.raises(ProxyRateLimitError) as exc:
|
||||
await _check(handler, _class_team("team-b"), saturation=0.52, priority="interactive")
|
||||
_held_to_class_share(exc.value)
|
||||
assert "Share: 500, Remaining: 0, Model saturation: 52.0%" in str(exc.value.detail)
|
||||
await _burn(handler, _class_team("team-a"), "interactive", 470, saturation=0.52)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_requests_sent_while_the_model_was_idle_charge_only_their_tokens_to_the_class_share(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
handler: Final = _borrowing_handler(monkeypatch, borrows=False, splits=True)
|
||||
await _burn(handler, _class_team("team-a"), "interactive", 20, saturation=0.0)
|
||||
for _ in range(5):
|
||||
await _burn(handler, _class_team("team-b"), "interactive", 99, saturation=0.02)
|
||||
await _burn(handler, _class_team("team-b"), "interactive", 5, saturation=0.52)
|
||||
with pytest.raises(ProxyRateLimitError) as exc:
|
||||
await _check(handler, _class_team("team-b"), saturation=0.52, priority="interactive")
|
||||
_held_to_class_share(exc.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_only_active_class_gets_the_whole_model_instead_of_its_weight(monkeypatch: pytest.MonkeyPatch):
|
||||
handler = _borrowing_handler(monkeypatch, borrows=False, splits=True)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue