mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): apply fairness settings after configured callbacks load
fairness_settings listed above callbacks in config.yaml made fairness register the limiter first and record it as its own, so disabling fairness later removed the limiter the config asked for. Fairness now applies once the litellm_settings loop finishes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
85760bef60
commit
36558fb30c
2 changed files with 41 additions and 2 deletions
|
|
@ -6112,8 +6112,8 @@ class ProxyConfig:
|
|||
from litellm.types.utils import PriorityReservationSettings
|
||||
|
||||
litellm.priority_reservation_settings = PriorityReservationSettings(**value)
|
||||
elif key == "fairness_settings":
|
||||
self._apply_fairness_settings_value(value, router)
|
||||
elif key == FAIRNESS_SETTINGS_KEY:
|
||||
continue
|
||||
elif key == "callbacks":
|
||||
initialize_callbacks_on_proxy(
|
||||
value=value,
|
||||
|
|
@ -6337,6 +6337,9 @@ class ProxyConfig:
|
|||
|
||||
reset_audit_log_callback_cache()
|
||||
_in_memory_loggers[:] = [cb for cb in _in_memory_loggers if not isinstance(cb, S3V2Logger)]
|
||||
fairness_value: Final = litellm_settings.get(FAIRNESS_SETTINGS_KEY)
|
||||
if fairness_value is not None:
|
||||
self._apply_fairness_settings_value(fairness_value, router)
|
||||
|
||||
if redis_usage_cache is None:
|
||||
env_coordination_redis_cache: Final = await self._init_coordination_redis_env_fallback(
|
||||
|
|
|
|||
|
|
@ -11891,6 +11891,42 @@ def test_fairness_settings_propagate_on_config_reload(monkeypatch):
|
|||
assert "dynamic_rate_limiter_v3" in litellm.callbacks
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabling_fairness_keeps_a_limiter_configured_after_fairness_settings_in_yaml(tmp_path, monkeypatch):
|
||||
"""The config file lists fairness_settings above callbacks, so if fairness were applied in
|
||||
key order it would register the limiter first and later remove the one the config asked for."""
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy.hooks.fairness_settings import _CONFIG_RESERVATION
|
||||
from litellm.types.proxy.fairness import FairnessSettings
|
||||
|
||||
monkeypatch.setattr(litellm, "fairness_settings", None)
|
||||
monkeypatch.setattr(litellm, "priority_reservation", None)
|
||||
monkeypatch.setattr(litellm, "priority_reservation_settings", None)
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
monkeypatch.setattr(_CONFIG_RESERVATION, "limiter_added_by_fairness", False)
|
||||
config_file = tmp_path / "config.yaml"
|
||||
config_file.write_text(
|
||||
yaml.dump(
|
||||
{
|
||||
"model_list": [{"model_name": "m", "litellm_params": {"model": "openai/m", "api_key": "k"}}],
|
||||
"litellm_settings": {
|
||||
"fairness_settings": {
|
||||
"enabled": True,
|
||||
"workload_classes": [{"name": "prod", "reserved_share": 0.5}],
|
||||
},
|
||||
"callbacks": ["dynamic_rate_limiter_v3"],
|
||||
},
|
||||
},
|
||||
sort_keys=False,
|
||||
)
|
||||
)
|
||||
|
||||
router, _, _ = await ps.ProxyConfig().load_config(router=None, config_file_path=str(config_file))
|
||||
ps.ProxyConfig._apply_fairness_settings_value(FairnessSettings(enabled=False), router)
|
||||
|
||||
assert "dynamic_rate_limiter_v3" in litellm.callbacks, litellm.callbacks
|
||||
|
||||
|
||||
def test_invalid_fairness_settings_row_is_ignored_on_config_reload(monkeypatch):
|
||||
"""A malformed row must not take down the reload or half-apply."""
|
||||
import litellm.proxy.proxy_server as ps
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue