fix(least-busy): release a pick whose call ends before reaching the provider

A call that fails or returns before the provider handoff (a 400 for an unsupported param, token counting, a proxy-gate rejection without a start log, batch create) now drops its reservation instead of decrementing a count it never added. Router.select picks and batch cancel no longer reserve, since neither makes a call that logs.
This commit is contained in:
mateo-berri 2026-10-29 18:02:26 -07:00
parent 854231f823
commit 67bc809393
3 changed files with 125 additions and 13 deletions

View file

@ -1387,6 +1387,7 @@ class Router(DeprecatedRouterMembers):
messages=messages,
input=input,
specific_deployment=specific_deployment,
reserve=False,
)
if kind == "healthy":
return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts
@ -8063,6 +8064,7 @@ class Router(DeprecatedRouterMembers):
messages=[{"role": "user", "content": "batch-api-fake-text"}],
specific_deployment=kwargs.pop("specific_deployment", None),
request_kwargs=kwargs,
reserve=False,
)
selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id")
@ -15695,6 +15697,7 @@ class Router(DeprecatedRouterMembers):
messages: list[dict[str, str]] | None = None,
input: str | list | None = None,
specific_deployment: bool | None = False,
reserve: bool = True,
):
"""
Async implementation of 'get_available_deployments'.
@ -15708,6 +15711,7 @@ class Router(DeprecatedRouterMembers):
messages=messages,
input=input,
specific_deployment=specific_deployment,
reserve=reserve,
)
async def _async_get_available_deployment(
@ -15717,6 +15721,7 @@ class Router(DeprecatedRouterMembers):
messages: list[dict[str, str]] | None,
input: str | list | None,
specific_deployment: bool | None,
reserve: bool,
):
if (
self._core_routing_strategy != "usage-based-routing-v2"
@ -15813,7 +15818,7 @@ class Router(DeprecatedRouterMembers):
messages=messages,
input=input,
request_kwargs=request_kwargs,
reserve=request_kwargs not in (None, {}),
reserve=reserve and request_kwargs not in (None, {}),
)
if deployment is None:
exception: Final = await async_raise_no_deployment_exception(

View file

@ -12,6 +12,7 @@ from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
@ -96,6 +97,12 @@ def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None:
return model_group, str(deployment_id)
def _counted_in_flight(kwargs: Mapping[str, object]) -> bool:
return kwargs.get("cache_hit") is not True and (
kwargs.get("api_call_start_time") is not None or LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL in kwargs
)
def _reservation_key(request_count_key: str) -> str:
return f"{request_count_key}:reservations"
@ -218,10 +225,7 @@ class LeastBusyLoggingHandler(CustomLogger):
def log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
if kwargs.get("cache_hit") is True:
self._release_reservation(kwargs)
else:
self._increment(kwargs, -1)
self._settle(kwargs)
if self.test_flag:
self.logged_success += 1
@ -229,7 +233,7 @@ class LeastBusyLoggingHandler(CustomLogger):
def log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._increment(kwargs, -1)
self._settle(kwargs)
if self.test_flag:
self.logged_failure += 1
@ -237,10 +241,7 @@ class LeastBusyLoggingHandler(CustomLogger):
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
if kwargs.get("cache_hit") is True:
self._release_reservation(kwargs)
else:
await self._async_increment(kwargs, -1)
await self._async_settle(kwargs)
if self.test_flag:
self.logged_success += 1
@ -248,7 +249,7 @@ class LeastBusyLoggingHandler(CustomLogger):
async def async_log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
await self._async_increment(kwargs, -1)
await self._async_settle(kwargs)
if self.test_flag:
self.logged_failure += 1
@ -282,6 +283,18 @@ class LeastBusyLoggingHandler(CustomLogger):
return self._select(healthy_deployments, keys, self._at_least_this_workers(keys, shared), reserve)
return self._select(healthy_deployments, keys, self._this_workers_counts(keys), reserve)
def _settle(self, kwargs: Mapping[str, object]) -> None:
if not _counted_in_flight(kwargs):
self._release_reservation(kwargs)
return
self._increment(kwargs, -1)
async def _async_settle(self, kwargs: Mapping[str, object]) -> None:
if not _counted_in_flight(kwargs):
self._release_reservation(kwargs)
return
await self._async_increment(kwargs, -1)
def _this_workers_counts(self, keys: tuple[str, ...]) -> tuple[int, ...]:
return _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys)

View file

@ -1,5 +1,6 @@
import asyncio
import contextlib
import datetime
import json
import logging
import threading
@ -8,6 +9,7 @@ from concurrent.futures import ThreadPoolExecutor
from typing import Final
import httpx
import openai
import pytest
import respx
@ -17,6 +19,7 @@ from litellm import Router
from litellm.caching.caching import DualCache
from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY, InMemoryCache
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.router_strategy.least_busy import (
IN_FLIGHT_COUNT_TTL_SECONDS,
@ -31,10 +34,14 @@ DEPLOYMENT_B: Final[dict[str, object]] = {"model_info": {"id": "dep-b"}}
HEALTHY: Final = [DEPLOYMENT_A, DEPLOYMENT_B]
def _call_kwargs(deployment_id: str) -> dict[str, object]:
def _routed_call_kwargs(deployment_id: str) -> dict[str, object]:
return {"litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}}
def _call_kwargs(deployment_id: str) -> dict[str, object]:
return {**_routed_call_kwargs(deployment_id), "api_call_start_time": datetime.datetime.now()}
class SharedRedisCounters:
"""Mirrors what Redis gives the handler: increments clamped at zero, a TTL set once when
the key is created, and ordered reads that raise rather than invent a value."""
@ -136,7 +143,8 @@ def _routed_through_litellm_metadata(deployment_id: str) -> dict[str, object]:
"metadata": {"tags": ["caller-tag"]},
"litellm_metadata": {"model_group": GROUP},
"model_info": {"id": deployment_id},
}
},
"api_call_start_time": datetime.datetime.now(),
}
@ -632,3 +640,89 @@ def test_a_cache_hit_leaves_requests_in_flight_on_its_deployment_counted() -> No
worker.log_success_event({**_call_kwargs("dep-a"), "cache_hit": True}, None, None, None)
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
@pytest.mark.parametrize("outcome", [LeastBusyLoggingHandler.log_success_event, LeastBusyLoggingHandler.log_failure_event])
def test_a_call_that_ends_before_reaching_the_provider_frees_its_pick(
outcome: Callable[[LeastBusyLoggingHandler, dict[str, object], None, None, None], None],
) -> None:
worker: Final = _worker(None)
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
outcome(worker, _routed_call_kwargs("dep-a"), None, None, None)
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
def test_a_call_the_proxy_rejected_after_logging_its_start_leaves_no_count_behind() -> None:
worker: Final = _worker(None)
rejected: Final = {**_routed_call_kwargs("dep-a"), LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True}
worker.log_pre_api_call(model="m", messages=[], kwargs=rejected)
worker.log_failure_event(rejected, None, None, None)
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
async def _drain_logging() -> None:
for _ in range(10):
await asyncio.sleep(0)
await GLOBAL_LOGGING_WORKER.flush()
def _batch_deployment(route: respx.Route) -> str:
return route.calls.last.request.headers["authorization"].removeprefix("Bearer key-")
_BATCH: Final = {
"id": "batch_least_busy",
"object": "batch",
"endpoint": "/v1/chat/completions",
"input_file_id": "file-least-busy",
"completion_window": "24h",
"status": "validating",
"created_at": 1,
}
_REJECTED_BATCH: Final = {"error": {"message": "Invalid input file", "type": "invalid_request_error"}}
@pytest.mark.asyncio
@pytest.mark.parametrize("provider_response", [httpx.Response(200, json=_BATCH), httpx.Response(400, json=_REJECTED_BATCH)])
async def test_a_batch_create_does_not_keep_its_deployment_busy(
provider_response: httpx.Response, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
route: Final = respx_mock.post("https://api.openai.com/v1/batches").mock(return_value=provider_response)
router: Final = _least_busy_router()
with contextlib.suppress(openai.BadRequestError):
await router.acreate_batch(
model=ROUTER_GROUP, completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file-least-busy"
)
await _drain_logging()
assert _router_pick(router) == _batch_deployment(route)
@pytest.mark.asyncio
async def test_a_batch_cancel_does_not_keep_its_deployment_busy(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
route: Final = respx_mock.post("https://api.openai.com/v1/batches/batch_least_busy/cancel").mock(
return_value=httpx.Response(200, json={**_BATCH, "status": "cancelling"})
)
router: Final = _least_busy_router()
await router.acancel_batch(model=ROUTER_GROUP, batch_id="batch_least_busy")
assert _router_pick(router) == _batch_deployment(route)
@pytest.mark.asyncio
async def test_picking_a_deployment_without_calling_it_does_not_keep_it_busy() -> None:
router: Final = _least_busy_router()
picked: Final = await router.select(ROUTER_GROUP, {"metadata": {}})
assert _router_pick(router) == picked["model_info"]["id"]