mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(least-busy): release a pick whose call ends before reaching the provider
A call that fails or returns before the provider handoff (a 400 for an unsupported param, token counting, a proxy-gate rejection without a start log, batch create) now drops its reservation instead of decrementing a count it never added. Router.select picks and batch cancel no longer reserve, since neither makes a call that logs.
This commit is contained in:
parent
854231f823
commit
67bc809393
3 changed files with 125 additions and 13 deletions
|
|
@ -1387,6 +1387,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
reserve=False,
|
||||
)
|
||||
if kind == "healthy":
|
||||
return await self._core_async_get_healthy_deployments( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the selection cores return untyped deployment dicts
|
||||
|
|
@ -8063,6 +8064,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=[{"role": "user", "content": "batch-api-fake-text"}],
|
||||
specific_deployment=kwargs.pop("specific_deployment", None),
|
||||
request_kwargs=kwargs,
|
||||
reserve=False,
|
||||
)
|
||||
|
||||
selected_deployment_id: Final = (deployment.get("model_info") or {}).get("id")
|
||||
|
|
@ -15695,6 +15697,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages: list[dict[str, str]] | None = None,
|
||||
input: str | list | None = None,
|
||||
specific_deployment: bool | None = False,
|
||||
reserve: bool = True,
|
||||
):
|
||||
"""
|
||||
Async implementation of 'get_available_deployments'.
|
||||
|
|
@ -15708,6 +15711,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
reserve=reserve,
|
||||
)
|
||||
|
||||
async def _async_get_available_deployment(
|
||||
|
|
@ -15717,6 +15721,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages: list[dict[str, str]] | None,
|
||||
input: str | list | None,
|
||||
specific_deployment: bool | None,
|
||||
reserve: bool,
|
||||
):
|
||||
if (
|
||||
self._core_routing_strategy != "usage-based-routing-v2"
|
||||
|
|
@ -15813,7 +15818,7 @@ class Router(DeprecatedRouterMembers):
|
|||
messages=messages,
|
||||
input=input,
|
||||
request_kwargs=request_kwargs,
|
||||
reserve=request_kwargs not in (None, {}),
|
||||
reserve=reserve and request_kwargs not in (None, {}),
|
||||
)
|
||||
if deployment is None:
|
||||
exception: Final = await async_raise_no_deployment_exception(
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from litellm._logging import verbose_router_logger
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.caching.redis_cache import log_redis_failure
|
||||
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
|
||||
|
|
@ -96,6 +97,12 @@ def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None:
|
|||
return model_group, str(deployment_id)
|
||||
|
||||
|
||||
def _counted_in_flight(kwargs: Mapping[str, object]) -> bool:
|
||||
return kwargs.get("cache_hit") is not True and (
|
||||
kwargs.get("api_call_start_time") is not None or LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL in kwargs
|
||||
)
|
||||
|
||||
|
||||
def _reservation_key(request_count_key: str) -> str:
|
||||
return f"{request_count_key}:reservations"
|
||||
|
||||
|
|
@ -218,10 +225,7 @@ class LeastBusyLoggingHandler(CustomLogger):
|
|||
def log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
if kwargs.get("cache_hit") is True:
|
||||
self._release_reservation(kwargs)
|
||||
else:
|
||||
self._increment(kwargs, -1)
|
||||
self._settle(kwargs)
|
||||
if self.test_flag:
|
||||
self.logged_success += 1
|
||||
|
||||
|
|
@ -229,7 +233,7 @@ class LeastBusyLoggingHandler(CustomLogger):
|
|||
def log_failure_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self._increment(kwargs, -1)
|
||||
self._settle(kwargs)
|
||||
if self.test_flag:
|
||||
self.logged_failure += 1
|
||||
|
||||
|
|
@ -237,10 +241,7 @@ class LeastBusyLoggingHandler(CustomLogger):
|
|||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
if kwargs.get("cache_hit") is True:
|
||||
self._release_reservation(kwargs)
|
||||
else:
|
||||
await self._async_increment(kwargs, -1)
|
||||
await self._async_settle(kwargs)
|
||||
if self.test_flag:
|
||||
self.logged_success += 1
|
||||
|
||||
|
|
@ -248,7 +249,7 @@ class LeastBusyLoggingHandler(CustomLogger):
|
|||
async def async_log_failure_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
await self._async_increment(kwargs, -1)
|
||||
await self._async_settle(kwargs)
|
||||
if self.test_flag:
|
||||
self.logged_failure += 1
|
||||
|
||||
|
|
@ -282,6 +283,18 @@ class LeastBusyLoggingHandler(CustomLogger):
|
|||
return self._select(healthy_deployments, keys, self._at_least_this_workers(keys, shared), reserve)
|
||||
return self._select(healthy_deployments, keys, self._this_workers_counts(keys), reserve)
|
||||
|
||||
def _settle(self, kwargs: Mapping[str, object]) -> None:
|
||||
if not _counted_in_flight(kwargs):
|
||||
self._release_reservation(kwargs)
|
||||
return
|
||||
self._increment(kwargs, -1)
|
||||
|
||||
async def _async_settle(self, kwargs: Mapping[str, object]) -> None:
|
||||
if not _counted_in_flight(kwargs):
|
||||
self._release_reservation(kwargs)
|
||||
return
|
||||
await self._async_increment(kwargs, -1)
|
||||
|
||||
def _this_workers_counts(self, keys: tuple[str, ...]) -> tuple[int, ...]:
|
||||
return _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
|
|
@ -8,6 +9,7 @@ from concurrent.futures import ThreadPoolExecutor
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
|
|
@ -17,6 +19,7 @@ from litellm import Router
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.in_memory_cache import DEFAULT_MAX_SIZE_IN_MEMORY, InMemoryCache
|
||||
from litellm.caching.redis_cache import RedisCircuitBreakerOpenError
|
||||
from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.router_strategy.least_busy import (
|
||||
IN_FLIGHT_COUNT_TTL_SECONDS,
|
||||
|
|
@ -31,10 +34,14 @@ DEPLOYMENT_B: Final[dict[str, object]] = {"model_info": {"id": "dep-b"}}
|
|||
HEALTHY: Final = [DEPLOYMENT_A, DEPLOYMENT_B]
|
||||
|
||||
|
||||
def _call_kwargs(deployment_id: str) -> dict[str, object]:
|
||||
def _routed_call_kwargs(deployment_id: str) -> dict[str, object]:
|
||||
return {"litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}}
|
||||
|
||||
|
||||
def _call_kwargs(deployment_id: str) -> dict[str, object]:
|
||||
return {**_routed_call_kwargs(deployment_id), "api_call_start_time": datetime.datetime.now()}
|
||||
|
||||
|
||||
class SharedRedisCounters:
|
||||
"""Mirrors what Redis gives the handler: increments clamped at zero, a TTL set once when
|
||||
the key is created, and ordered reads that raise rather than invent a value."""
|
||||
|
|
@ -136,7 +143,8 @@ def _routed_through_litellm_metadata(deployment_id: str) -> dict[str, object]:
|
|||
"metadata": {"tags": ["caller-tag"]},
|
||||
"litellm_metadata": {"model_group": GROUP},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
},
|
||||
"api_call_start_time": datetime.datetime.now(),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -632,3 +640,89 @@ def test_a_cache_hit_leaves_requests_in_flight_on_its_deployment_counted() -> No
|
|||
worker.log_success_event({**_call_kwargs("dep-a"), "cache_hit": True}, None, None, None)
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
|
||||
|
||||
|
||||
@pytest.mark.parametrize("outcome", [LeastBusyLoggingHandler.log_success_event, LeastBusyLoggingHandler.log_failure_event])
|
||||
def test_a_call_that_ends_before_reaching_the_provider_frees_its_pick(
|
||||
outcome: Callable[[LeastBusyLoggingHandler, dict[str, object], None, None, None], None],
|
||||
) -> None:
|
||||
worker: Final = _worker(None)
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY, reserve=True) is DEPLOYMENT_A
|
||||
|
||||
outcome(worker, _routed_call_kwargs("dep-a"), None, None, None)
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
|
||||
|
||||
|
||||
def test_a_call_the_proxy_rejected_after_logging_its_start_leaves_no_count_behind() -> None:
|
||||
worker: Final = _worker(None)
|
||||
rejected: Final = {**_routed_call_kwargs("dep-a"), LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: True}
|
||||
|
||||
worker.log_pre_api_call(model="m", messages=[], kwargs=rejected)
|
||||
worker.log_failure_event(rejected, None, None, None)
|
||||
|
||||
assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
|
||||
|
||||
|
||||
async def _drain_logging() -> None:
|
||||
for _ in range(10):
|
||||
await asyncio.sleep(0)
|
||||
await GLOBAL_LOGGING_WORKER.flush()
|
||||
|
||||
|
||||
def _batch_deployment(route: respx.Route) -> str:
|
||||
return route.calls.last.request.headers["authorization"].removeprefix("Bearer key-")
|
||||
|
||||
|
||||
_BATCH: Final = {
|
||||
"id": "batch_least_busy",
|
||||
"object": "batch",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"input_file_id": "file-least-busy",
|
||||
"completion_window": "24h",
|
||||
"status": "validating",
|
||||
"created_at": 1,
|
||||
}
|
||||
_REJECTED_BATCH: Final = {"error": {"message": "Invalid input file", "type": "invalid_request_error"}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("provider_response", [httpx.Response(200, json=_BATCH), httpx.Response(400, json=_REJECTED_BATCH)])
|
||||
async def test_a_batch_create_does_not_keep_its_deployment_busy(
|
||||
provider_response: httpx.Response, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/batches").mock(return_value=provider_response)
|
||||
router: Final = _least_busy_router()
|
||||
|
||||
with contextlib.suppress(openai.BadRequestError):
|
||||
await router.acreate_batch(
|
||||
model=ROUTER_GROUP, completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file-least-busy"
|
||||
)
|
||||
await _drain_logging()
|
||||
|
||||
assert _router_pick(router) == _batch_deployment(route)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_batch_cancel_does_not_keep_its_deployment_busy(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/batches/batch_least_busy/cancel").mock(
|
||||
return_value=httpx.Response(200, json={**_BATCH, "status": "cancelling"})
|
||||
)
|
||||
router: Final = _least_busy_router()
|
||||
|
||||
await router.acancel_batch(model=ROUTER_GROUP, batch_id="batch_least_busy")
|
||||
|
||||
assert _router_pick(router) == _batch_deployment(route)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_picking_a_deployment_without_calling_it_does_not_keep_it_busy() -> None:
|
||||
router: Final = _least_busy_router()
|
||||
|
||||
picked: Final = await router.select(ROUTER_GROUP, {"metadata": {}})
|
||||
|
||||
assert _router_pick(router) == picked["model_info"]["id"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue