diff --git a/litellm/scheduler.py b/litellm/scheduler.py index 73196d24e4d..e4ff6e3a287 100644 --- a/litellm/scheduler.py +++ b/litellm/scheduler.py @@ -101,6 +101,8 @@ class Scheduler: timeout: float, get_healthy_deployments: Callable[[], Awaitable[Sequence[object]]], ) -> None: + if len(await get_healthy_deployments()) > 0: + return try: await self.add_request(request=request) end_time: Final = time.monotonic() + timeout diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index e0968e9b2cf..8688fc1ba5d 100644 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -227,6 +227,23 @@ async def test_wait_for_turn_during_cooldown_survives_the_queue_key_expiring(): assert await scheduler.get_queue("sched-model") == [] +@pytest.mark.asyncio +async def test_wait_for_turn_never_queues_a_request_a_healthy_deployment_can_take(): + scheduler: Final = Scheduler() + + async def one_healthy_deployment_seen_with_an_empty_queue() -> Sequence[object]: + queue: Final = await scheduler.get_queue("sched-model") + assert queue == [], f"the admitted request was queued while a deployment was healthy: {queue}" + return ({"model_info": {"id": "a"}},) + + await scheduler.wait_for_turn( + request=FlowItem(priority=1, request_id="admitted", model_name="sched-model"), + timeout=5, + get_healthy_deployments=one_healthy_deployment_seen_with_an_empty_queue, + ) + + assert await scheduler.get_queue("sched-model") == [] + class _JsonRoundTripRedisCache: def __init__(self) -> None: