fix(proxy): refund fairness reservations on proxy-side rejection and never bill a reserved request twice

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-26 09:32:27 +00:00
parent 1f08c3b517
commit f19f5a1210
4 changed files with 109 additions and 3 deletions

View file

@ -44,6 +44,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
call_id_from_callback_kwargs,
claim_request_stash_for_data,
get_or_create_request_stash,
get_request_stash,
get_request_stash_for_call,
)
from litellm.proxy.hooks.rate_limiter_utils import (
@ -986,7 +987,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
)
stash: Final = get_request_stash_for_call(call_id_from_callback_kwargs(kwargs))
if stash is not None and await self._settle_reservation(stash, total_tokens, litellm_parent_otel_span):
if stash is not None and stash.dynamic_reserved_tokens > 0:
await self._settle_reservation(stash, total_tokens, litellm_parent_otel_span)
return
bypassed: Final = stash.dynamic_admission_bypassed if stash is not None else not self.enforcing
@ -1072,6 +1074,22 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
except Exception as e:
verbose_proxy_logger.exception("Error refunding dynamic rate limiter reservation: %s", e)
async def async_post_call_failure_hook(
self,
request_data: dict,
original_exception: Exception,
user_api_key_dict: UserAPIKeyAuth,
traceback_str: str | None = None,
) -> None:
stash: Final = get_request_stash()
if stash is None:
return
recovered_tokens: Final = self.v3_limiter.recovered_partial_usage_tokens(request_data)[0]
try:
await self._settle_reservation(stash, recovered_tokens, user_api_key_dict.parent_otel_span)
except Exception as e:
verbose_proxy_logger.exception("Error refunding dynamic rate limiter reservation: %s", e)
async def fairness_status(self, models: Sequence[str]) -> tuple[ModelFairnessStatus, ...]:
router: Final = self.llm_router
if router is None:

View file

@ -5,6 +5,7 @@ from typing import Final, Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
DEFAULT_POOL_NAME: Final = "default"
RESERVED_WORKLOAD_CLASS_NAMES: Final = frozenset({DEFAULT_POOL_NAME, "default_pool"})
FAIRNESS_SETTINGS_KEY: Final = "fairness_settings"
MAX_QUEUE_WAIT_HEADER: Final = "x-litellm-max-queue-wait"
MAX_QUEUE_WAIT_SECONDS: Final = 600.0
@ -44,8 +45,11 @@ class FairnessSettings(BaseModel):
names: Final = tuple(workload_class.name for workload_class in self.workload_classes)
if len(names) != len(frozenset(names)):
raise ValueError("workload class names must be unique")
if DEFAULT_POOL_NAME in names:
raise ValueError(f"{DEFAULT_POOL_NAME!r} is reserved for keys and teams without a workload class")
if RESERVED_WORKLOAD_CLASS_NAMES & frozenset(names):
raise ValueError(
f"workload class names {sorted(RESERVED_WORKLOAD_CLASS_NAMES)!r} are reserved for keys and teams "
"without a workload class"
)
total_share: Final = (
sum(workload_class.reserved_share for workload_class in self.workload_classes) + self.default_reserved_share
)

View file

@ -2519,3 +2519,79 @@ async def test_failed_settlement_write_leaves_reservation_open_for_refund(monkey
assert still_open
assert writes == ["attempt", "attempt"]
assert await _model_tokens(handler, dual_cache, model) == 0
@pytest.mark.asyncio
async def test_post_call_failure_hook_refunds_reservation_taken_at_admission(monkeypatch):
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass
model = "fairness-proxy-reject-model"
_enable_fairness(
monkeypatch,
FairnessSettings(enabled=True, workload_classes=(WorkloadClass(name="prod", reserved_share=0.5),)),
)
dual_cache = DualCache()
handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)
handler.update_variables(llm_router=_fairness_router(model, tpm=100_000))
data = {
"model": model,
"litellm_call_id": "proxy-reject",
"messages": [{"role": "user", "content": "a downstream hook rejects this after admission"}],
"max_tokens": 100,
}
settle_mock = AsyncMock()
monkeypatch.setattr(handler.v3_limiter, "async_increment_reservation_aware_tokens", settle_mock)
monkeypatch.setattr(handler.v3_limiter, "recovered_partial_usage_tokens", lambda *args, **kwargs: (0, 0, 0))
async def reserve_then_reject() -> tuple[int, bool]:
await handler.async_pre_call_hook(
user_api_key_dict=_prod_user(), cache=dual_cache, data=data, call_type="completion"
)
reserved = get_or_create_request_stash().dynamic_reserved_tokens
await handler.async_post_call_failure_hook(
request_data=data,
original_exception=Exception("rejected by a later hook"),
user_api_key_dict=_prod_user(),
)
return reserved, get_or_create_request_stash().dynamic_reservation_settled
reserved, settled = await asyncio.create_task(reserve_then_reject())
assert reserved > 0
assert settled
settle_mock.assert_awaited_once()
@pytest.mark.asyncio
async def test_success_event_never_bills_an_already_settled_reservation(monkeypatch):
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass
from litellm.types.utils import ModelResponse, Usage
model = "fairness-double-bill-model"
_enable_fairness(
monkeypatch,
FairnessSettings(enabled=True, workload_classes=(WorkloadClass(name="prod", reserved_share=0.5),)),
)
dual_cache = DualCache()
handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)
handler.update_variables(llm_router=_fairness_router(model, tpm=100_000))
legacy_increment = AsyncMock()
monkeypatch.setattr(handler.v3_limiter, "async_increment_tokens_with_ttl_preservation", legacy_increment)
async def already_settled_then_success() -> None:
stash = get_or_create_request_stash()
stash.dynamic_reserved_tokens = 40
stash.dynamic_token_scopes = frozenset({("model_saturation_check", model)})
stash.dynamic_reservation_settled = True
await handler.async_log_success_event(
kwargs=_success_kwargs(model, "double-bill", "prod"),
response_obj=ModelResponse(
model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10)
),
start_time=None,
end_time=None,
)
await asyncio.create_task(already_settled_then_success())
legacy_increment.assert_not_awaited()

View file

@ -265,6 +265,14 @@ def test_settings_reject_duplicate_and_reserved_class_names() -> None:
FairnessSettings(workload_classes=(WorkloadClass(name="default", reserved_share=0.1),))
def test_default_pool_is_a_reserved_workload_class_name() -> None:
with pytest.raises(ValidationError, match="reserved"):
FairnessSettings(
enabled=True,
workload_classes=(WorkloadClass(name="default_pool", reserved_share=0.1),),
)
def test_settings_reject_shares_that_exceed_full_capacity_including_default_pool() -> None:
classes: Final = (
WorkloadClass(name="production", reserved_share=0.6),