mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(router): honor AllowedFails=0 policy for cooldown-on-first-failure
This commit is contained in:
parent
cd6e8cdf23
commit
c2a6c9ca95
2 changed files with 34 additions and 4 deletions
|
|
@ -372,11 +372,11 @@ def should_cooldown_based_on_allowed_fails_policy(
|
|||
- True if fails exceed the allowed limit (should cooldown)
|
||||
- False if fails are within the allowed limit (should not cooldown)
|
||||
"""
|
||||
allowed_fails_from_policy = litellm_router_instance.get_allowed_fails_from_policy(
|
||||
exception=original_exception,
|
||||
)
|
||||
allowed_fails = (
|
||||
litellm_router_instance.get_allowed_fails_from_policy(
|
||||
exception=original_exception,
|
||||
)
|
||||
or litellm_router_instance.allowed_fails
|
||||
allowed_fails_from_policy if allowed_fails_from_policy is not None else litellm_router_instance.allowed_fails
|
||||
)
|
||||
cooldown_time = litellm_router_instance.cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
|
||||
|
||||
|
|
|
|||
|
|
@ -214,6 +214,36 @@ class TestHealthCheckCooldownIntegration:
|
|||
)
|
||||
assert result is True
|
||||
|
||||
def test_allowed_fails_policy_zero_cooldowns_on_first_failure(self):
|
||||
"""Regression for #32425.
|
||||
|
||||
AllowedFails=0 means cooldown on the very first failure. The value 0 was
|
||||
previously discarded by an `or` fallback (`policy_value or self.allowed_fails`),
|
||||
so the generic allowed_fails default was used instead and the first failure
|
||||
did not cool the deployment down.
|
||||
"""
|
||||
from litellm.router_utils.cooldown_handlers import (
|
||||
should_cooldown_based_on_allowed_fails_policy,
|
||||
)
|
||||
|
||||
router = Router(
|
||||
model_list=[_make_model("deploy-1"), _make_model("deploy-2", "gpt-5")],
|
||||
allowed_fails_policy=AllowedFailsPolicy(RateLimitErrorAllowedFails=0),
|
||||
# A permissive generic threshold that must NOT be used when the policy says 0.
|
||||
allowed_fails=100,
|
||||
)
|
||||
|
||||
rate_limit_exc = litellm.RateLimitError(
|
||||
message="rate limited", model="gpt-4", llm_provider="openai"
|
||||
)
|
||||
|
||||
result = should_cooldown_based_on_allowed_fails_policy(
|
||||
litellm_router_instance=router,
|
||||
deployment="deploy-1",
|
||||
original_exception=rate_limit_exc,
|
||||
)
|
||||
assert result is True
|
||||
|
||||
def test_health_check_failure_falls_back_to_allowed_fails(self):
|
||||
"""When policy has no matching field, fall back to generic allowed_fails."""
|
||||
from litellm.router_utils.cooldown_handlers import (
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue