mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): refund fairness reservations on proxy-side rejection and never bill a reserved request twice
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
1f08c3b517
commit
f19f5a1210
4 changed files with 109 additions and 3 deletions
|
|
@ -44,6 +44,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import (
|
|||
call_id_from_callback_kwargs,
|
||||
claim_request_stash_for_data,
|
||||
get_or_create_request_stash,
|
||||
get_request_stash,
|
||||
get_request_stash_for_call,
|
||||
)
|
||||
from litellm.proxy.hooks.rate_limiter_utils import (
|
||||
|
|
@ -986,7 +987,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
|
|||
)
|
||||
|
||||
stash: Final = get_request_stash_for_call(call_id_from_callback_kwargs(kwargs))
|
||||
if stash is not None and await self._settle_reservation(stash, total_tokens, litellm_parent_otel_span):
|
||||
if stash is not None and stash.dynamic_reserved_tokens > 0:
|
||||
await self._settle_reservation(stash, total_tokens, litellm_parent_otel_span)
|
||||
return
|
||||
|
||||
bypassed: Final = stash.dynamic_admission_bypassed if stash is not None else not self.enforcing
|
||||
|
|
@ -1072,6 +1074,22 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error refunding dynamic rate limiter reservation: %s", e)
|
||||
|
||||
async def async_post_call_failure_hook(
|
||||
self,
|
||||
request_data: dict,
|
||||
original_exception: Exception,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
traceback_str: str | None = None,
|
||||
) -> None:
|
||||
stash: Final = get_request_stash()
|
||||
if stash is None:
|
||||
return
|
||||
recovered_tokens: Final = self.v3_limiter.recovered_partial_usage_tokens(request_data)[0]
|
||||
try:
|
||||
await self._settle_reservation(stash, recovered_tokens, user_api_key_dict.parent_otel_span)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error refunding dynamic rate limiter reservation: %s", e)
|
||||
|
||||
async def fairness_status(self, models: Sequence[str]) -> tuple[ModelFairnessStatus, ...]:
|
||||
router: Final = self.llm_router
|
||||
if router is None:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import Final, Literal
|
|||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
DEFAULT_POOL_NAME: Final = "default"
|
||||
RESERVED_WORKLOAD_CLASS_NAMES: Final = frozenset({DEFAULT_POOL_NAME, "default_pool"})
|
||||
FAIRNESS_SETTINGS_KEY: Final = "fairness_settings"
|
||||
MAX_QUEUE_WAIT_HEADER: Final = "x-litellm-max-queue-wait"
|
||||
MAX_QUEUE_WAIT_SECONDS: Final = 600.0
|
||||
|
|
@ -44,8 +45,11 @@ class FairnessSettings(BaseModel):
|
|||
names: Final = tuple(workload_class.name for workload_class in self.workload_classes)
|
||||
if len(names) != len(frozenset(names)):
|
||||
raise ValueError("workload class names must be unique")
|
||||
if DEFAULT_POOL_NAME in names:
|
||||
raise ValueError(f"{DEFAULT_POOL_NAME!r} is reserved for keys and teams without a workload class")
|
||||
if RESERVED_WORKLOAD_CLASS_NAMES & frozenset(names):
|
||||
raise ValueError(
|
||||
f"workload class names {sorted(RESERVED_WORKLOAD_CLASS_NAMES)!r} are reserved for keys and teams "
|
||||
"without a workload class"
|
||||
)
|
||||
total_share: Final = (
|
||||
sum(workload_class.reserved_share for workload_class in self.workload_classes) + self.default_reserved_share
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2519,3 +2519,79 @@ async def test_failed_settlement_write_leaves_reservation_open_for_refund(monkey
|
|||
assert still_open
|
||||
assert writes == ["attempt", "attempt"]
|
||||
assert await _model_tokens(handler, dual_cache, model) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_refunds_reservation_taken_at_admission(monkeypatch):
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
|
||||
from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass
|
||||
|
||||
model = "fairness-proxy-reject-model"
|
||||
_enable_fairness(
|
||||
monkeypatch,
|
||||
FairnessSettings(enabled=True, workload_classes=(WorkloadClass(name="prod", reserved_share=0.5),)),
|
||||
)
|
||||
dual_cache = DualCache()
|
||||
handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)
|
||||
handler.update_variables(llm_router=_fairness_router(model, tpm=100_000))
|
||||
data = {
|
||||
"model": model,
|
||||
"litellm_call_id": "proxy-reject",
|
||||
"messages": [{"role": "user", "content": "a downstream hook rejects this after admission"}],
|
||||
"max_tokens": 100,
|
||||
}
|
||||
settle_mock = AsyncMock()
|
||||
monkeypatch.setattr(handler.v3_limiter, "async_increment_reservation_aware_tokens", settle_mock)
|
||||
monkeypatch.setattr(handler.v3_limiter, "recovered_partial_usage_tokens", lambda *args, **kwargs: (0, 0, 0))
|
||||
|
||||
async def reserve_then_reject() -> tuple[int, bool]:
|
||||
await handler.async_pre_call_hook(
|
||||
user_api_key_dict=_prod_user(), cache=dual_cache, data=data, call_type="completion"
|
||||
)
|
||||
reserved = get_or_create_request_stash().dynamic_reserved_tokens
|
||||
await handler.async_post_call_failure_hook(
|
||||
request_data=data,
|
||||
original_exception=Exception("rejected by a later hook"),
|
||||
user_api_key_dict=_prod_user(),
|
||||
)
|
||||
return reserved, get_or_create_request_stash().dynamic_reservation_settled
|
||||
|
||||
reserved, settled = await asyncio.create_task(reserve_then_reject())
|
||||
assert reserved > 0
|
||||
assert settled
|
||||
settle_mock.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success_event_never_bills_an_already_settled_reservation(monkeypatch):
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
|
||||
from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
model = "fairness-double-bill-model"
|
||||
_enable_fairness(
|
||||
monkeypatch,
|
||||
FairnessSettings(enabled=True, workload_classes=(WorkloadClass(name="prod", reserved_share=0.5),)),
|
||||
)
|
||||
dual_cache = DualCache()
|
||||
handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)
|
||||
handler.update_variables(llm_router=_fairness_router(model, tpm=100_000))
|
||||
legacy_increment = AsyncMock()
|
||||
monkeypatch.setattr(handler.v3_limiter, "async_increment_tokens_with_ttl_preservation", legacy_increment)
|
||||
|
||||
async def already_settled_then_success() -> None:
|
||||
stash = get_or_create_request_stash()
|
||||
stash.dynamic_reserved_tokens = 40
|
||||
stash.dynamic_token_scopes = frozenset({("model_saturation_check", model)})
|
||||
stash.dynamic_reservation_settled = True
|
||||
await handler.async_log_success_event(
|
||||
kwargs=_success_kwargs(model, "double-bill", "prod"),
|
||||
response_obj=ModelResponse(
|
||||
model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10)
|
||||
),
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
|
||||
await asyncio.create_task(already_settled_then_success())
|
||||
legacy_increment.assert_not_awaited()
|
||||
|
|
|
|||
|
|
@ -265,6 +265,14 @@ def test_settings_reject_duplicate_and_reserved_class_names() -> None:
|
|||
FairnessSettings(workload_classes=(WorkloadClass(name="default", reserved_share=0.1),))
|
||||
|
||||
|
||||
def test_default_pool_is_a_reserved_workload_class_name() -> None:
|
||||
with pytest.raises(ValidationError, match="reserved"):
|
||||
FairnessSettings(
|
||||
enabled=True,
|
||||
workload_classes=(WorkloadClass(name="default_pool", reserved_share=0.1),),
|
||||
)
|
||||
|
||||
|
||||
def test_settings_reject_shares_that_exceed_full_capacity_including_default_pool() -> None:
|
||||
classes: Final = (
|
||||
WorkloadClass(name="production", reserved_share=0.6),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue