From 67bc8093939b9dfe3cb2a50648769ff9d8861b3c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:02:26 -0700 Subject: [PATCH] fix(least-busy): release a pick whose call ends before reaching the provider A call that fails or returns before the provider handoff (a 400 for an unsupported param, token counting, a proxy-gate rejection without a start log, batch create) now drops its reservation instead of decrementing a count it never added. Router.select picks and batch cancel no longer reserve, since neither makes a call that logs. --- litellm/router.py | 7 +- litellm/router_strategy/least_busy.py | 33 +++++-- tests/unit/router_strategy/test_least_busy.py | 98 ++++++++++++++++++- 3 files changed, 125 insertions(+), 13 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index b3a9524fd0e..8c095360301 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1387,6 +1387,7 @@ class Router(DeprecatedRouterMembers): messages=messages, input=input, specific_deployment=specific_deployment, + reserve=False, ) if kind == "healthy": return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts @@ -8063,6 +8064,7 @@ class Router(DeprecatedRouterMembers): messages=[{"role": "user", "content": "batch-api-fake-text"}], specific_deployment=kwargs.pop("specific_deployment", None), request_kwargs=kwargs, + reserve=False, ) selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id") @@ -15695,6 +15697,7 @@ class Router(DeprecatedRouterMembers): messages: list[dict[str, str]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + reserve: bool = True, ): """ Async implementation of 'get_available_deployments'. @@ -15708,6 +15711,7 @@ class Router(DeprecatedRouterMembers): messages=messages, input=input, specific_deployment=specific_deployment, + reserve=reserve, ) async def _async_get_available_deployment( @@ -15717,6 +15721,7 @@ class Router(DeprecatedRouterMembers): messages: list[dict[str, str]] | None, input: str | list | None, specific_deployment: bool | None, + reserve: bool, ): if ( self._core_routing_strategy != "usage-based-routing-v2" @@ -15813,7 +15818,7 @@ class Router(DeprecatedRouterMembers): messages=messages, input=input, request_kwargs=request_kwargs, - reserve=request_kwargs not in (None, {}), + reserve=reserve and request_kwargs not in (None, {}), ) if deployment is None: exception: Final = await async_raise_no_deployment_exception( diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py index 1efd4a6e7a5..f920e427d41 100644 --- a/litellm/router_strategy/least_busy.py +++ b/litellm/router_strategy/least_busy.py @@ -12,6 +12,7 @@ from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import log_redis_failure +from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type @@ -96,6 +97,12 @@ def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None: return model_group, str(deployment_id) +def _counted_in_flight(kwargs: Mapping[str, object]) -> bool: + return kwargs.get("cache_hit") is not True and ( + kwargs.get("api_call_start_time") is not None or LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL in kwargs + ) + + def _reservation_key(request_count_key: str) -> str: return f"{request_count_key}:reservations" @@ -218,10 +225,7 @@ class LeastBusyLoggingHandler(CustomLogger): def log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - if kwargs.get("cache_hit") is True: - self._release_reservation(kwargs) - else: - self._increment(kwargs, -1) + self._settle(kwargs) if self.test_flag: self.logged_success += 1 @@ -229,7 +233,7 @@ class LeastBusyLoggingHandler(CustomLogger): def log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - self._increment(kwargs, -1) + self._settle(kwargs) if self.test_flag: self.logged_failure += 1 @@ -237,10 +241,7 @@ class LeastBusyLoggingHandler(CustomLogger): async def async_log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - if kwargs.get("cache_hit") is True: - self._release_reservation(kwargs) - else: - await self._async_increment(kwargs, -1) + await self._async_settle(kwargs) if self.test_flag: self.logged_success += 1 @@ -248,7 +249,7 @@ class LeastBusyLoggingHandler(CustomLogger): async def async_log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object ) -> None: - await self._async_increment(kwargs, -1) + await self._async_settle(kwargs) if self.test_flag: self.logged_failure += 1 @@ -282,6 +283,18 @@ class LeastBusyLoggingHandler(CustomLogger): return self._select(healthy_deployments, keys, self._at_least_this_workers(keys, shared), reserve) return self._select(healthy_deployments, keys, self._this_workers_counts(keys), reserve) + def _settle(self, kwargs: Mapping[str, object]) -> None: + if not _counted_in_flight(kwargs): + self._release_reservation(kwargs) + return + self._increment(kwargs, -1) + + async def _async_settle(self, kwargs: Mapping[str, object]) -> None: + if not _counted_in_flight(kwargs): + self._release_reservation(kwargs) + return + await self._async_increment(kwargs, -1) + def _this_workers_counts(self, keys: tuple[str, ...]) -> tuple[int, ...]: return _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys) diff --git a/tests/unit/router_strategy/test_least_busy.py b/tests/unit/router_strategy/test_least_busy.py index a0be7741aa9..4b7cbe253bd 100644 --- a/tests/unit/router_strategy/test_least_busy.py +++ b/tests/unit/router_strategy/test_least_busy.py @@ -1,5 +1,6 @@ import asyncio import contextlib +import datetime import json import logging import threading @@ -8,6 +9,7 @@ from concurrent.futures import ThreadPoolExecutor from typing import Final import httpx +import openai import pytest import respx @@ -17,6 +19,7 @@ from litellm import Router from litellm.caching.caching import DualCache from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY, InMemoryCache from litellm.caching.redis_cache import RedisCircuitBreakerOpenError +from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.router_strategy.least_busy import ( IN_FLIGHT_COUNT_TTL_SECONDS, @@ -31,10 +34,14 @@ DEPLOYMENT_B: Final[dict[str, object]] = {"model_info": {"id": "dep-b"}} HEALTHY: Final = [DEPLOYMENT_A, DEPLOYMENT_B] -def _call_kwargs(deployment_id: str) -> dict[str, object]: +def _routed_call_kwargs(deployment_id: str) -> dict[str, object]: return {"litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}} +def _call_kwargs(deployment_id: str) -> dict[str, object]: + return {**_routed_call_kwargs(deployment_id), "api_call_start_time": datetime.datetime.now()} + + class SharedRedisCounters: """Mirrors what Redis gives the handler: increments clamped at zero, a TTL set once when the key is created, and ordered reads that raise rather than invent a value.""" @@ -136,7 +143,8 @@ def _routed_through_litellm_metadata(deployment_id: str) -> dict[str, object]: "metadata": {"tags": ["caller-tag"]}, "litellm_metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}, - } + }, + "api_call_start_time": datetime.datetime.now(), } @@ -632,3 +640,89 @@ def test_a_cache_hit_leaves_requests_in_flight_on_its_deployment_counted() -> No worker.log_success_event({**_call_kwargs("dep-a"), "cache_hit": True}, None, None, None) assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B + + +@pytest.mark.parametrize("outcome", [LeastBusyLoggingHandler.log_success_event, LeastBusyLoggingHandler.log_failure_event]) +def test_a_call_that_ends_before_reaching_the_provider_frees_its_pick( + outcome: Callable[[LeastBusyLoggingHandler, dict[str, object], None, None, None], None], +) -> None: + worker: Final = _worker(None) + assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A + + outcome(worker, _routed_call_kwargs("dep-a"), None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + +def test_a_call_the_proxy_rejected_after_logging_its_start_leaves_no_count_behind() -> None: + worker: Final = _worker(None) + rejected: Final = {**_routed_call_kwargs("dep-a"), LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True} + + worker.log_pre_api_call(model="m", messages=[], kwargs=rejected) + worker.log_failure_event(rejected, None, None, None) + + assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A + + +async def _drain_logging() -> None: + for _ in range(10): + await asyncio.sleep(0) + await GLOBAL_LOGGING_WORKER.flush() + + +def _batch_deployment(route: respx.Route) -> str: + return route.calls.last.request.headers["authorization"].removeprefix("Bearer key-") + + +_BATCH: Final = { + "id": "batch_least_busy", + "object": "batch", + "endpoint": "/v1/chat/completions", + "input_file_id": "file-least-busy", + "completion_window": "24h", + "status": "validating", + "created_at": 1, +} +_REJECTED_BATCH: Final = {"error": {"message": "Invalid input file", "type": "invalid_request_error"}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider_response", [httpx.Response(200, json=_BATCH), httpx.Response(400, json=_REJECTED_BATCH)]) +async def test_a_batch_create_does_not_keep_its_deployment_busy( + provider_response: httpx.Response, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/batches").mock(return_value=provider_response) + router: Final = _least_busy_router() + + with contextlib.suppress(openai.BadRequestError): + await router.acreate_batch( + model=ROUTER_GROUP, completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file-least-busy" + ) + await _drain_logging() + + assert _router_pick(router) == _batch_deployment(route) + + +@pytest.mark.asyncio +async def test_a_batch_cancel_does_not_keep_its_deployment_busy( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/batches/batch_least_busy/cancel").mock( + return_value=httpx.Response(200, json={**_BATCH, "status": "cancelling"}) + ) + router: Final = _least_busy_router() + + await router.acancel_batch(model=ROUTER_GROUP, batch_id="batch_least_busy") + + assert _router_pick(router) == _batch_deployment(route) + + +@pytest.mark.asyncio +async def test_picking_a_deployment_without_calling_it_does_not_keep_it_busy() -> None: + router: Final = _least_busy_router() + + picked: Final = await router.select(ROUTER_GROUP, {"metadata": {}}) + + assert _router_pick(router) == picked["model_info"]["id"]