mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): unregister the fairness-owned limiter when fairness is disabled live
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
a0d2e7fcc0
commit
eeeeea76f5
2 changed files with 51 additions and 4 deletions
|
|
@ -37,6 +37,7 @@ class _ConfigReservation:
|
|||
|
||||
priority_reservation: Mapping[str, float | PriorityReservationDict] | None = None
|
||||
settings: PriorityReservationSettings | None = None
|
||||
limiter_added_by_fairness: bool = False
|
||||
|
||||
|
||||
_CONFIG_RESERVATION: Final = _ConfigReservation()
|
||||
|
|
@ -53,10 +54,19 @@ def parse_fairness_settings(value: object) -> FairnessSettings | None:
|
|||
|
||||
|
||||
def active_fairness_limiter() -> _PROXY_DynamicRateLimitHandlerV3 | None:
|
||||
if not _limiter_registered():
|
||||
return None
|
||||
limiter: Final = get_custom_logger_compatible_class(_LIMITER_CALLBACK)
|
||||
return limiter if isinstance(limiter, _PROXY_DynamicRateLimitHandlerV3) else None
|
||||
|
||||
|
||||
def _limiter_registered() -> bool:
|
||||
return any(
|
||||
callback == _LIMITER_CALLBACK or isinstance(callback, _PROXY_DynamicRateLimitHandlerV3)
|
||||
for callback in litellm.callbacks
|
||||
)
|
||||
|
||||
|
||||
def apply_fairness_settings(
|
||||
settings: FairnessSettings,
|
||||
internal_usage_cache: DualCache | None,
|
||||
|
|
@ -69,10 +79,12 @@ def apply_fairness_settings(
|
|||
if was_enabled:
|
||||
litellm.priority_reservation = _CONFIG_RESERVATION.priority_reservation
|
||||
litellm.priority_reservation_settings = _CONFIG_RESERVATION.settings or PriorityReservationSettings()
|
||||
_unregister_fairness_limiter()
|
||||
return
|
||||
if not was_enabled:
|
||||
_CONFIG_RESERVATION.priority_reservation = litellm.priority_reservation
|
||||
_CONFIG_RESERVATION.settings = litellm.priority_reservation_settings
|
||||
_CONFIG_RESERVATION.limiter_added_by_fairness = not _limiter_registered()
|
||||
litellm.priority_reservation = settings.reserved_shares()
|
||||
litellm.priority_reservation_settings = PriorityReservationSettings(
|
||||
default_priority=settings.default_reserved_share,
|
||||
|
|
@ -83,10 +95,7 @@ def apply_fairness_settings(
|
|||
|
||||
|
||||
def _ensure_limiter_registered(internal_usage_cache: DualCache | None, llm_router: Router | None) -> None:
|
||||
already_registered: Final = any(
|
||||
callback == _LIMITER_CALLBACK or isinstance(callback, _PROXY_DynamicRateLimitHandlerV3)
|
||||
for callback in litellm.callbacks
|
||||
)
|
||||
already_registered: Final = _limiter_registered()
|
||||
existing: Final = active_fairness_limiter()
|
||||
if existing is None and (llm_router is None or internal_usage_cache is None):
|
||||
if not already_registered:
|
||||
|
|
@ -105,3 +114,14 @@ def _ensure_limiter_registered(internal_usage_cache: DualCache | None, llm_route
|
|||
initialized.update_variables(llm_router=llm_router)
|
||||
if not already_registered:
|
||||
litellm.logging_callback_manager.add_litellm_callback(initialized)
|
||||
|
||||
|
||||
def _unregister_fairness_limiter() -> None:
|
||||
if not _CONFIG_RESERVATION.limiter_added_by_fairness:
|
||||
return
|
||||
_CONFIG_RESERVATION.limiter_added_by_fairness = False
|
||||
manager: Final = litellm.logging_callback_manager
|
||||
limiter: Final = active_fairness_limiter()
|
||||
manager.remove_callback_from_list_by_object(litellm.callbacks, _LIMITER_CALLBACK, require_self=False)
|
||||
if limiter is not None:
|
||||
manager.remove_callback_from_all_lists(limiter)
|
||||
|
|
|
|||
|
|
@ -208,6 +208,33 @@ def test_disabling_fairness_restores_config_file_reservations(
|
|||
assert litellm.priority_reservation_settings == PriorityReservationSettings(saturation_threshold=0.9)
|
||||
|
||||
|
||||
def _limiter_callbacks() -> tuple[object, ...]:
|
||||
return tuple(
|
||||
callback
|
||||
for callback in litellm.callbacks
|
||||
if callback == "dynamic_rate_limiter_v3" or isinstance(callback, _PROXY_DynamicRateLimitHandlerV3)
|
||||
)
|
||||
|
||||
|
||||
def test_disabling_fairness_unregisters_the_limiter_it_added(
|
||||
isolated_globals: _FakeProxyConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
apply_fairness_settings(_settings(), internal_usage_cache=DualCache(), llm_router=Router(model_list=[]))
|
||||
assert len(_limiter_callbacks()) == 1, litellm.callbacks
|
||||
apply_fairness_settings(FairnessSettings(enabled=False), internal_usage_cache=None, llm_router=None)
|
||||
assert _limiter_callbacks() == (), litellm.callbacks
|
||||
|
||||
|
||||
def test_disabling_fairness_keeps_the_config_file_limiter(
|
||||
isolated_globals: _FakeProxyConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "callbacks", ["dynamic_rate_limiter_v3"])
|
||||
apply_fairness_settings(_settings(), internal_usage_cache=None, llm_router=None)
|
||||
apply_fairness_settings(FairnessSettings(enabled=False), internal_usage_cache=None, llm_router=None)
|
||||
assert _limiter_callbacks() == ("dynamic_rate_limiter_v3",), litellm.callbacks
|
||||
|
||||
|
||||
def test_settings_reject_duplicate_and_reserved_class_names() -> None:
|
||||
with pytest.raises(ValidationError, match="unique"):
|
||||
FairnessSettings(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue