test(rate-limiting): lock in that fastest_response's winner now releases every reserved unit

Reserving one unit per real branch (not just the whole dispatch)
also resolves the previously-accepted undercount for
abatch_completion_fastest_response, since the one winner's terminal
event releases everything pending together. Adds a regression test
so this doesn't regress back into a leak if release logic changes
again.
This commit is contained in:
Deepanshu 2026-09-03 16:52:53 -04:00
parent 231a2c6a0e
commit dc95401149

View file

@ -1441,6 +1441,53 @@ async def test_fastest_response_batch_also_reserves_one_unit_per_branch(time_con
)
@pytest.mark.asyncio
async def test_fastest_response_winner_releases_every_unit_the_race_reserved(time_controller, monkeypatch):
"""abatch_completion_fastest_response's cancelled losers never fire a
terminal event, but the one winner's own event must still release every
unit the whole race reserved, not just its own share -- this used to be
an accepted undercount (reserving only 1 unit for the whole race); now
that admission reserves one per real branch, release must give all of
it back together, or the fix here would trade an undercount for a
leak."""
monkeypatch.setattr(
litellm,
"global_tag_rate_limits",
{
"concurrency_limits": {
"limits": [{"name": "conc", "tag_id": "end_user_id", "limit": 2, "period_seconds": 60}]
}
},
)
hook = _make_hook(time_controller)
await hook.async_pre_call_hook(
user_api_key_dict=_key(),
cache=DualCache(),
data={
**_data(["end_user_id:u1"], call_id="call-1"),
"model": "model-a,model-b",
"fastest_response": True,
},
call_type="acompletion",
)
kwargs = {"litellm_call_id": "call-1", "metadata": {"tags": ["end_user_id:u1"]}}
await hook.async_log_success_event(kwargs=kwargs, response_obj=None, start_time=0, end_time=0)
await asyncio.sleep(0)
result = await hook.async_pre_call_hook(
user_api_key_dict=_key(),
cache=DualCache(),
data={
**_data(["end_user_id:u1"], call_id="call-2"),
"model": "model-c,model-d",
"fastest_response": True,
},
call_type="acompletion",
)
assert result is not None
@pytest.mark.asyncio
async def test_batch_dispatch_release_is_safe_across_both_real_terminal_events(time_controller, monkeypatch):
"""has_logged_async_success and has_logged_async_failure gate