From 0933fd9efb1d8723e2f714e7c8406663f0e01a2f Mon Sep 17 00:00:00 2001 From: Deepanshu Date: Wed, 26 Aug 2026 16:26:11 -0400 Subject: [PATCH] docs(proxy): document why a deployment-scoped breach rejects the whole hop bugbot flagged this as a bug (per-deployment-scoped limits should filter the over-limit deployment out of healthy_deployments and let a sibling serve, not reject the whole routing attempt). This is deliberate, documented design intent from the original plan: an earlier draft considered filter-and-retry semantics and rejected it, since rejecting the whole hop is simpler and avoids a caller silently succeeding against a deployment whose limit configuration they didn't intend to satisfy. Adding that reasoning as an inline comment so it doesn't get re-flagged as a bug on a future review. --- .../proxy/hooks/model_based_tag_rate_limits_hook.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/litellm/proxy/hooks/model_based_tag_rate_limits_hook.py b/litellm/proxy/hooks/model_based_tag_rate_limits_hook.py index e68dbc81208..f292caf8cc3 100644 --- a/litellm/proxy/hooks/model_based_tag_rate_limits_hook.py +++ b/litellm/proxy/hooks/model_based_tag_rate_limits_hook.py @@ -739,6 +739,17 @@ def _classify_check( now: float, key_alias: str | None, ) -> _ClassifiedCheck | None: + # A breach of a deployment-scoped check (this function's own + # `deployment_scope is not None` branch, and every one of its callers' + # `_raise_over_limit`) deliberately rejects the whole routing attempt for + # this hop, not just the deployment(s) that own it -- it does not filter + # them out of `healthy_deployments` and let a sibling in the same group + # serve instead. An earlier design considered filter-and-retry-sibling + # semantics (matching how native tag routing filters candidates rather + # than rejecting the hop) and deliberately did not adopt it: rejecting + # the whole hop is simpler, and avoids a caller silently succeeding + # against a deployment whose limit configuration they didn't intend to + # satisfy. if configured_limit.deployment_scope is not None and not ( present_deployment_ids & frozenset(configured_limit.deployment_scope) ):