From 01b75cdfd9f7d0aff4b6755307ea48c284865142 Mon Sep 17 00:00:00 2001 From: Deepanshu Date: Thu, 27 Aug 2026 07:00:55 -0400 Subject: [PATCH] fix(rate-limiting): stop a mid-chain fallback rejection from bypassing an apply_to_models cap (ported from #38347) _pre_call_with_fallbacks only checked the original exception for cross_model_scope before starting the fallback loop; a later fallback attempt's own cross_model_scope rejection was swallowed by the loop's plain except-continue and the next, unlisted fallback model silently served the request instead (baefd27677). --- litellm/proxy/common_request_processing.py | 14 +++- .../proxy/test_common_request_processing.py | 64 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index d46f0ac8e60..f98ff8e8e62 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2045,7 +2045,19 @@ class ProxyBaseLLMRequestProcessing: route_type=route_type, llm_router=llm_router, ) - except ProxyRateLimitError: + except ProxyRateLimitError as fallback_exc: + # A fallback attempt's own rejection can carry the + # identical cross_model_scope marker (this fallback + # model is itself covered by the same apply_to_models + # chain-wide cap) -- continuing to the next fallback + # would silently serve the request through a model + # outside that cap, defeating it just as much as not + # checking the original exception would. + if ( + isinstance(fallback_exc.detail, Mapping) + and fallback_exc.detail.get("cross_model_scope") is True + ): + raise continue except BaseException: self.data["model"] = original_model diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 8b286a7420e..d83000fd5dd 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -5535,6 +5535,70 @@ class TestPreCallWithFallbacksOnLocalRateLimit: assert call_count == 1 assert processor.data["model"] == primary_model + @pytest.mark.asyncio + async def test_cross_model_scoped_rejection_mid_chain_stops_further_fallback_attempts(self): + """ + The check above only covers the ORIGINAL exception, before the + fallback loop starts. A later fallback attempt's own rejection was + never checked the same way -- the loop's ``except + ProxyRateLimitError: continue`` swallowed it and moved on to the next + fallback model. If a chain-wide ``apply_to_models`` cap covers both + the primary model and the first fallback, and a second fallback model + isn't covered, this let the second fallback silently serve the + request the cap was meant to block. The original (non-scoped) + rejection enters the loop normally; the FIRST fallback's own + rejection carries ``cross_model_scope=True`` and must stop the loop + immediately, never reaching the second fallback. + """ + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + primary_model = "opus-chain" + + processor = ProxyBaseLLMRequestProcessing(data={"model": primary_model}) + + attempted_models: list[str] = [] + + async def mock_pre_call_logic(**kwargs): + attempted_models.append(processor.data["model"]) + if processor.data["model"] == primary_model: + # Original attempt: a plain, non-scoped rejection (e.g. a + # per-deployment limit), not the chain-wide cap itself. + raise ProxyRateLimitError(detail={"error": "tag_rate_limit_exceeded"}, headers={"retry-after": "30"}) + if processor.data["model"] == "sonnet-chain": + # First fallback: rejected by the SAME chain-wide cap. + raise ProxyRateLimitError( + detail={"error": "tag_rate_limit_exceeded", "cross_model_scope": True}, + headers={"retry-after": "30"}, + ) + raise AssertionError(f"must not attempt a second fallback model: {processor.data['model']}") + + mock_router = MagicMock() + mock_router.fallbacks = [{"opus-chain": ["sonnet-chain", "haiku-chain"]}] + + with patch.object(processor, "common_processing_pre_call_logic", side_effect=mock_pre_call_logic): + with pytest.raises(ProxyRateLimitError) as exc_info: + await processor._pre_call_with_fallbacks( + request=MagicMock(), + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=MagicMock(router_settings=None), + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=primary_model, + route_type="acompletion", + llm_router=mock_router, + ) + + assert attempted_models == [primary_model, "sonnet-chain"] + assert exc_info.value.detail.get("cross_model_scope") is True + assert processor.data["model"] == primary_model + @pytest.mark.asyncio async def test_real_parallel_request_limiter_model_tpm_limit_triggers_fallback(self): """