From f19f5a12109ab04768f8a7bcb104af972031e2fd Mon Sep 17 00:00:00 2001 From: yassin Date: Sat, 26 Sep 2026 09:32:27 +0000 Subject: [PATCH] fix(proxy): refund fairness reservations on proxy-side rejection and never bill a reserved request twice Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/hooks/dynamic_rate_limiter_v3.py | 20 ++++- litellm/types/proxy/fairness.py | 8 +- .../hooks/test_dynamic_rate_limiter_v3.py | 76 +++++++++++++++++++ .../test_fairness_endpoints.py | 8 ++ 4 files changed, 109 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index cc03bedff5d..1c0f8274a37 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -44,6 +44,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( call_id_from_callback_kwargs, claim_request_stash_for_data, get_or_create_request_stash, + get_request_stash, get_request_stash_for_call, ) from litellm.proxy.hooks.rate_limiter_utils import ( @@ -986,7 +987,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ) stash: Final = get_request_stash_for_call(call_id_from_callback_kwargs(kwargs)) - if stash is not None and await self._settle_reservation(stash, total_tokens, litellm_parent_otel_span): + if stash is not None and stash.dynamic_reserved_tokens > 0: + await self._settle_reservation(stash, total_tokens, litellm_parent_otel_span) return bypassed: Final = stash.dynamic_admission_bypassed if stash is not None else not self.enforcing @@ -1072,6 +1074,22 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): except Exception as e: verbose_proxy_logger.exception("Error refunding dynamic rate limiter reservation: %s", e) + async def async_post_call_failure_hook( + self, + request_data: dict, + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: str | None = None, + ) -> None: + stash: Final = get_request_stash() + if stash is None: + return + recovered_tokens: Final = self.v3_limiter.recovered_partial_usage_tokens(request_data)[0] + try: + await self._settle_reservation(stash, recovered_tokens, user_api_key_dict.parent_otel_span) + except Exception as e: + verbose_proxy_logger.exception("Error refunding dynamic rate limiter reservation: %s", e) + async def fairness_status(self, models: Sequence[str]) -> tuple[ModelFairnessStatus, ...]: router: Final = self.llm_router if router is None: diff --git a/litellm/types/proxy/fairness.py b/litellm/types/proxy/fairness.py index 4b71ad7a5d1..ced9023749c 100644 --- a/litellm/types/proxy/fairness.py +++ b/litellm/types/proxy/fairness.py @@ -5,6 +5,7 @@ from typing import Final, Literal from pydantic import BaseModel, ConfigDict, Field, model_validator DEFAULT_POOL_NAME: Final = "default" +RESERVED_WORKLOAD_CLASS_NAMES: Final = frozenset({DEFAULT_POOL_NAME, "default_pool"}) FAIRNESS_SETTINGS_KEY: Final = "fairness_settings" MAX_QUEUE_WAIT_HEADER: Final = "x-litellm-max-queue-wait" MAX_QUEUE_WAIT_SECONDS: Final = 600.0 @@ -44,8 +45,11 @@ class FairnessSettings(BaseModel): names: Final = tuple(workload_class.name for workload_class in self.workload_classes) if len(names) != len(frozenset(names)): raise ValueError("workload class names must be unique") - if DEFAULT_POOL_NAME in names: - raise ValueError(f"{DEFAULT_POOL_NAME!r} is reserved for keys and teams without a workload class") + if RESERVED_WORKLOAD_CLASS_NAMES & frozenset(names): + raise ValueError( + f"workload class names {sorted(RESERVED_WORKLOAD_CLASS_NAMES)!r} are reserved for keys and teams " + "without a workload class" + ) total_share: Final = ( sum(workload_class.reserved_share for workload_class in self.workload_classes) + self.default_reserved_share ) diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 7f1170d9cc3..80c6cf2b65d 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -2519,3 +2519,79 @@ async def test_failed_settlement_write_leaves_reservation_open_for_refund(monkey assert still_open assert writes == ["attempt", "attempt"] assert await _model_tokens(handler, dual_cache, model) == 0 + + +@pytest.mark.asyncio +async def test_post_call_failure_hook_refunds_reservation_taken_at_admission(monkeypatch): + from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash + from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass + + model = "fairness-proxy-reject-model" + _enable_fairness( + monkeypatch, + FairnessSettings(enabled=True, workload_classes=(WorkloadClass(name="prod", reserved_share=0.5),)), + ) + dual_cache = DualCache() + handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache) + handler.update_variables(llm_router=_fairness_router(model, tpm=100_000)) + data = { + "model": model, + "litellm_call_id": "proxy-reject", + "messages": [{"role": "user", "content": "a downstream hook rejects this after admission"}], + "max_tokens": 100, + } + settle_mock = AsyncMock() + monkeypatch.setattr(handler.v3_limiter, "async_increment_reservation_aware_tokens", settle_mock) + monkeypatch.setattr(handler.v3_limiter, "recovered_partial_usage_tokens", lambda *args, **kwargs: (0, 0, 0)) + + async def reserve_then_reject() -> tuple[int, bool]: + await handler.async_pre_call_hook( + user_api_key_dict=_prod_user(), cache=dual_cache, data=data, call_type="completion" + ) + reserved = get_or_create_request_stash().dynamic_reserved_tokens + await handler.async_post_call_failure_hook( + request_data=data, + original_exception=Exception("rejected by a later hook"), + user_api_key_dict=_prod_user(), + ) + return reserved, get_or_create_request_stash().dynamic_reservation_settled + + reserved, settled = await asyncio.create_task(reserve_then_reject()) + assert reserved > 0 + assert settled + settle_mock.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_success_event_never_bills_an_already_settled_reservation(monkeypatch): + from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash + from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass + from litellm.types.utils import ModelResponse, Usage + + model = "fairness-double-bill-model" + _enable_fairness( + monkeypatch, + FairnessSettings(enabled=True, workload_classes=(WorkloadClass(name="prod", reserved_share=0.5),)), + ) + dual_cache = DualCache() + handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache) + handler.update_variables(llm_router=_fairness_router(model, tpm=100_000)) + legacy_increment = AsyncMock() + monkeypatch.setattr(handler.v3_limiter, "async_increment_tokens_with_ttl_preservation", legacy_increment) + + async def already_settled_then_success() -> None: + stash = get_or_create_request_stash() + stash.dynamic_reserved_tokens = 40 + stash.dynamic_token_scopes = frozenset({("model_saturation_check", model)}) + stash.dynamic_reservation_settled = True + await handler.async_log_success_event( + kwargs=_success_kwargs(model, "double-bill", "prod"), + response_obj=ModelResponse( + model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10) + ), + start_time=None, + end_time=None, + ) + + await asyncio.create_task(already_settled_then_success()) + legacy_increment.assert_not_awaited() diff --git a/tests/test_litellm/proxy/management_endpoints/test_fairness_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_fairness_endpoints.py index fc82fde9af7..97878920000 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_fairness_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_fairness_endpoints.py @@ -265,6 +265,14 @@ def test_settings_reject_duplicate_and_reserved_class_names() -> None: FairnessSettings(workload_classes=(WorkloadClass(name="default", reserved_share=0.1),)) +def test_default_pool_is_a_reserved_workload_class_name() -> None: + with pytest.raises(ValidationError, match="reserved"): + FairnessSettings( + enabled=True, + workload_classes=(WorkloadClass(name="default_pool", reserved_share=0.1),), + ) + + def test_settings_reject_shares_that_exceed_full_capacity_including_default_pool() -> None: classes: Final = ( WorkloadClass(name="production", reserved_share=0.6),