From dff08dcb55b35ff9346445b0ba3a733692e249f1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:16:45 +0000 Subject: [PATCH 1/7] fix(proxy): retry rate-limit fallbacks from a pristine request snapshot Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 38 ++- .../proxy/test_common_request_processing.py | 240 ++++++++++++++++++ 2 files changed, 264 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index e3a2b892721..255bf3ebda2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -32,7 +32,11 @@ from litellm.constants import ( UNSAFE_PROXY_RESPONSE_HEADERS, ) from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket, is_expected_client_error +from litellm.litellm_core_utils.core_helpers import ( + get_or_create_metadata_bucket, + independent_snapshot, + is_expected_client_error, +) from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, @@ -2034,6 +2038,21 @@ class ProxyBaseLLMRequestProcessing: ) -> tuple[dict, LiteLLMLoggingObj]: from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + original_model: Final = self.data.get("model") + fallback_models: Final = ( + self._resolve_fallback_models( + model=original_model, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + ) + if original_model + and isinstance(original_model, str) + and llm_router + and not self.data.get("disable_fallbacks") + else None + ) + pristine: Final = independent_snapshot(self.data) if fallback_models else None + try: return await self.common_processing_pre_call_logic( request=request, @@ -2052,16 +2071,7 @@ class ProxyBaseLLMRequestProcessing: llm_router=llm_router, ) except ProxyRateLimitError as original_exc: - original_model: Final = self.data.get("model") - if not original_model or not llm_router or self.data.get("disable_fallbacks"): - raise - - fallback_models: Final = self._resolve_fallback_models( - model=original_model, - llm_router=llm_router, - user_api_key_dict=user_api_key_dict, - ) - if not fallback_models: + if not fallback_models or pristine is None: raise verbose_proxy_logger.info( @@ -2074,7 +2084,7 @@ class ProxyBaseLLMRequestProcessing: for fallback_model in fallback_models: if fallback_model == original_model: continue - self.data["model"] = fallback_model + self.data = {**independent_snapshot(pristine), "model": fallback_model} try: return await self.common_processing_pre_call_logic( request=request, @@ -2095,10 +2105,10 @@ class ProxyBaseLLMRequestProcessing: except ProxyRateLimitError: continue except BaseException: - self.data["model"] = original_model + self.data = pristine raise - self.data["model"] = original_model + self.data = pristine raise original_exc def _resolve_fallback_models( diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 50b26577e5c..89e0799ba16 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -6235,6 +6235,246 @@ class TestPreCallWithFallbacksOnLocalRateLimit: call_type="acompletion", ) + @pytest.mark.asyncio + async def test_fallback_retries_from_pristine_request_data(self): + import threading + + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + primary_model = "gpt-4" + fallback_model = "gpt-3.5-turbo" + + processor = ProxyBaseLLMRequestProcessing( + data={ + "model": primary_model, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"tags": ["a"]}, + } + ) + + metadata_at_entry = [] + + async def mock_pre_call_logic(**kwargs): + copy.deepcopy(processor.data["metadata"]) + metadata_at_entry.append(dict(processor.data["metadata"])) + processor.data["metadata"]["litellm_parent_otel_span"] = threading.RLock() + processor.data["litellm_logging_obj"] = object() + if processor.data.get("model") == primary_model: + raise ProxyRateLimitError( + detail="TPM limit exceeded for gpt-4", + headers={"retry-after": "30"}, + ) + return processor.data, MagicMock() + + mock_router = MagicMock() + mock_router.fallbacks = [{primary_model: [fallback_model]}] + + with patch.object( + processor, + "common_processing_pre_call_logic", + side_effect=mock_pre_call_logic, + ): + data, logging_obj = await processor._pre_call_with_fallbacks( + request=MagicMock(), + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=MagicMock(router_settings=None), + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=primary_model, + route_type="acompletion", + llm_router=mock_router, + ) + + assert processor.data["model"] == fallback_model + assert metadata_at_entry[1] == {"tags": ["a"]} + + @pytest.mark.asyncio + async def test_exhausted_fallbacks_restore_pristine_request_data(self): + import threading + + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + primary_model = "gpt-4" + original_data = { + "model": primary_model, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"tags": ["a"]}, + } + processor = ProxyBaseLLMRequestProcessing(data=copy.deepcopy(original_data)) + + async def mock_pre_call_logic(**kwargs): + processor.data["metadata"]["litellm_parent_otel_span"] = threading.RLock() + processor.data["litellm_logging_obj"] = object() + raise ProxyRateLimitError( + detail=f"TPM limit exceeded for {processor.data.get('model')}", + headers={"retry-after": "30"}, + ) + + mock_router = MagicMock() + mock_router.fallbacks = [{primary_model: ["gpt-3.5-turbo"]}] + + with patch.object( + processor, + "common_processing_pre_call_logic", + side_effect=mock_pre_call_logic, + ): + with pytest.raises(ProxyRateLimitError, match="gpt-4"): + await processor._pre_call_with_fallbacks( + request=MagicMock(), + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=MagicMock(router_settings=None), + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=primary_model, + route_type="acompletion", + llm_router=mock_router, + ) + + assert processor.data == original_data + + @pytest.mark.asyncio + async def test_real_add_litellm_data_to_request_rerun_with_otel_span_falls_back(self): + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.proxy_server import ProxyConfig + + trace.set_tracer_provider(TracerProvider()) + + primary_model = "gpt-4" + fallback_model = "gpt-3.5-turbo" + + request_mock = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.path = "/v1/chat/completions" + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + user_api_key_dict = UserAPIKeyAuth( + parent_otel_span=trace.get_tracer("x").start_span("s"), + api_key="hashed-key", + user_id="u1", + team_id="t1", + metadata={}, + team_metadata={}, + team_member_tpm_limit=1000, + ) + + processor = ProxyBaseLLMRequestProcessing( + data={ + "model": primary_model, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"tags": ["a"]}, + } + ) + + async def real_add_litellm_data_pre_call(**kwargs): + await add_litellm_data_to_request( + data=processor.data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=ProxyConfig(), + general_settings={}, + version="test", + ) + if processor.data.get("model") == primary_model: + raise ProxyRateLimitError( + detail="TPM limit exceeded for gpt-4", + headers={"retry-after": "30"}, + ) + return processor.data, MagicMock() + + mock_router = MagicMock() + mock_router.fallbacks = [{primary_model: [fallback_model]}] + + with patch.object( + processor, + "common_processing_pre_call_logic", + side_effect=real_add_litellm_data_pre_call, + ): + data, logging_obj = await processor._pre_call_with_fallbacks( + request=request_mock, + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=user_api_key_dict, + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=primary_model, + route_type="acompletion", + llm_router=mock_router, + ) + + assert processor.data["model"] == fallback_model + + @pytest.mark.asyncio + async def test_no_fallbacks_skips_snapshot(self): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + processor = ProxyBaseLLMRequestProcessing(data={"model": "gpt-4"}) + + async def mock_pre_call_logic(**kwargs): + raise ProxyRateLimitError( + detail="TPM limit exceeded", + headers={"retry-after": "30"}, + ) + + mock_router = MagicMock() + mock_router.fallbacks = None + + with patch("litellm.proxy.common_request_processing.independent_snapshot") as snapshot_mock: + with patch.object( + processor, + "common_processing_pre_call_logic", + side_effect=mock_pre_call_logic, + ): + with pytest.raises(ProxyRateLimitError): + await processor._pre_call_with_fallbacks( + request=MagicMock(), + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=MagicMock(router_settings=None), + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model="gpt-4", + route_type="acompletion", + llm_router=mock_router, + ) + + snapshot_mock.assert_not_called() + class _RecordingSuccessLogger(CustomLogger): def __init__(self): From c064e576ee31ed05b067462412c6c234d364c8fd Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:29:20 +0000 Subject: [PATCH 2/7] fix(proxy): tolerate missing router_settings and non-list fallbacks in fallback resolution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 4 ++-- tests/test_litellm/proxy/test_common_request_processing.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 255bf3ebda2..9927320d795 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2121,14 +2121,14 @@ class ProxyBaseLLMRequestProcessing: fallbacks = None - key_router_settings: Final = user_api_key_dict.router_settings + key_router_settings: Final = getattr(user_api_key_dict, "router_settings", None) if isinstance(key_router_settings, dict) and "fallbacks" in key_router_settings: fallbacks = key_router_settings["fallbacks"] if fallbacks is None: fallbacks = llm_router.fallbacks - if not fallbacks: + if not isinstance(fallbacks, list) or not fallbacks: return None fallback_model_group, generic_fallback_idx = get_fallback_model_group( diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 89e0799ba16..4beb73b3a81 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -6449,7 +6449,9 @@ class TestPreCallWithFallbacksOnLocalRateLimit: mock_router = MagicMock() mock_router.fallbacks = None - with patch("litellm.proxy.common_request_processing.independent_snapshot") as snapshot_mock: + with patch( # test-quality-ok: spying the snapshot seam is the only observable check that the no-fallback path skips it + "litellm.proxy.common_request_processing.independent_snapshot" + ) as snapshot_mock: with patch.object( processor, "common_processing_pre_call_logic", From 431dcce6a72ddcb6ba73c56848e1b5e0477b7699 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:46:44 +0000 Subject: [PATCH 3/7] test(proxy): give pre-call mocks real router_settings and fallbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 4 ++-- .../test_response_polling_pre_call_checks.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9927320d795..255bf3ebda2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2121,14 +2121,14 @@ class ProxyBaseLLMRequestProcessing: fallbacks = None - key_router_settings: Final = getattr(user_api_key_dict, "router_settings", None) + key_router_settings: Final = user_api_key_dict.router_settings if isinstance(key_router_settings, dict) and "fallbacks" in key_router_settings: fallbacks = key_router_settings["fallbacks"] if fallbacks is None: fallbacks = llm_router.fallbacks - if not isinstance(fallbacks, list) or not fallbacks: + if not fallbacks: return None fallback_model_group, generic_fallback_idx = get_fallback_model_group( diff --git a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py index 459834d0fd2..38f087f51ca 100644 --- a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py +++ b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py @@ -48,10 +48,10 @@ class TestSkipPreCallLogic: await processor.base_process_llm_request( request=MagicMock(spec=Request), fastapi_response=MagicMock(spec=Response), - user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth, router_settings=None), route_type="aresponses", proxy_logging_obj=mock_proxy_logging, - llm_router=MagicMock(), + llm_router=MagicMock(fallbacks=None), general_settings={}, proxy_config=MagicMock(), skip_pre_call_logic=True, @@ -87,10 +87,10 @@ class TestSkipPreCallLogic: await processor.base_process_llm_request( request=MagicMock(spec=Request), fastapi_response=MagicMock(spec=Response), - user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth, router_settings=None), route_type="aresponses", proxy_logging_obj=mock_proxy_logging, - llm_router=MagicMock(), + llm_router=MagicMock(fallbacks=None), general_settings={}, proxy_config=MagicMock(), ) From 2b6184d76867fd38a990d2df124b7d1cd808ca6c Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 16 Sep 2026 07:08:24 +0000 Subject: [PATCH 4/7] fix(proxy): resolve rate-limit fallbacks after model normalization and retry from a client-request snapshot The fallback retry in _pre_call_with_fallbacks re-entered common_processing_pre_call_logic with data already enriched by the first pass, so add_litellm_data_to_request deep-copied a metadata dict holding the live OTel span and the request failed with a 500 (cannot pickle '_thread.RLock') instead of the intended 429 or fallback. Capture the configured fallbacks and a snapshot of the client request before the first pass, look up the fallback chain by the normalized model group after the limiter raises, and run each fallback attempt on a fresh copy of that snapshot. Replaces the mock-heavy tests with a rig that runs the real v3 limiter and a live OTel span through the proxy_logging_obj seam Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 62 ++-- .../test_response_polling_pre_call_checks.py | 8 +- .../proxy/test_common_request_processing.py | 347 +++++++----------- 3 files changed, 166 insertions(+), 251 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 3c8bb9b3c92..7c9a296965c 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2066,20 +2066,12 @@ class ProxyBaseLLMRequestProcessing: ) -> tuple[dict, LiteLLMLoggingObj]: from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError - original_model: Final = self.data.get("model") - fallback_models: Final = ( - self._resolve_fallback_models( - model=original_model, - llm_router=llm_router, - user_api_key_dict=user_api_key_dict, - ) - if original_model - and isinstance(original_model, str) - and llm_router - and not self.data.get("disable_fallbacks") + configured_fallbacks: Final = ( + self._configured_fallbacks(llm_router=llm_router, user_api_key_dict=user_api_key_dict) + if llm_router is not None and not self.data.get("disable_fallbacks") else None ) - pristine: Final = independent_snapshot(self.data) if fallback_models else None + pristine: Final = independent_snapshot(self.data) if configured_fallbacks else None try: return await self.common_processing_pre_call_logic( @@ -2099,7 +2091,16 @@ class ProxyBaseLLMRequestProcessing: llm_router=llm_router, ) except ProxyRateLimitError as original_exc: - if not fallback_models or pristine is None: + rate_limited_data: Final = self.data + original_model: Final = rate_limited_data.get("model") + if pristine is None or not configured_fallbacks or not isinstance(original_model, str): + raise + + fallback_models: Final = self._resolve_fallback_models( + model=original_model, + fallbacks=configured_fallbacks, + ) + if not fallback_models: raise verbose_proxy_logger.info( @@ -2133,39 +2134,30 @@ class ProxyBaseLLMRequestProcessing: except ProxyRateLimitError: continue except BaseException: - self.data = pristine + self.data = rate_limited_data raise - self.data = pristine + self.data = rate_limited_data raise original_exc - def _resolve_fallback_models( - self, - model: str, - llm_router: Router, - user_api_key_dict: UserAPIKeyAuth, - ) -> list | None: - from litellm.router_utils.fallback_event_handlers import get_fallback_model_group - - fallbacks = None - + @staticmethod + def _configured_fallbacks(llm_router: Router, user_api_key_dict: UserAPIKeyAuth) -> list | None: key_router_settings: Final = user_api_key_dict.router_settings - if isinstance(key_router_settings, dict) and "fallbacks" in key_router_settings: - fallbacks = key_router_settings["fallbacks"] + key_fallbacks: Final = key_router_settings.get("fallbacks") if isinstance(key_router_settings, dict) else None + fallbacks: Final = key_fallbacks if key_fallbacks is not None else llm_router.fallbacks + return fallbacks if isinstance(fallbacks, list) and fallbacks else None - if fallbacks is None: - fallbacks = llm_router.fallbacks - - if not fallbacks: - return None + @staticmethod + def _resolve_fallback_models(model: str, fallbacks: list) -> list | None: + from litellm.router_utils.fallback_event_handlers import get_fallback_model_group fallback_model_group, generic_fallback_idx = get_fallback_model_group( fallbacks=fallbacks, model_group=model, ) - if fallback_model_group is None and generic_fallback_idx is not None: - fallback_model_group = fallbacks[generic_fallback_idx]["*"] - return fallback_model_group + if fallback_model_group is not None: + return fallback_model_group + return fallbacks[generic_fallback_idx]["*"] if generic_fallback_idx is not None else None @staticmethod def _get_model_id_from_response(hidden_params: Mapping[str, object], data: Mapping[str, object]) -> str: diff --git a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py index 38f087f51ca..459834d0fd2 100644 --- a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py +++ b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py @@ -48,10 +48,10 @@ class TestSkipPreCallLogic: await processor.base_process_llm_request( request=MagicMock(spec=Request), fastapi_response=MagicMock(spec=Response), - user_api_key_dict=MagicMock(spec=UserAPIKeyAuth, router_settings=None), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), route_type="aresponses", proxy_logging_obj=mock_proxy_logging, - llm_router=MagicMock(fallbacks=None), + llm_router=MagicMock(), general_settings={}, proxy_config=MagicMock(), skip_pre_call_logic=True, @@ -87,10 +87,10 @@ class TestSkipPreCallLogic: await processor.base_process_llm_request( request=MagicMock(spec=Request), fastapi_response=MagicMock(spec=Response), - user_api_key_dict=MagicMock(spec=UserAPIKeyAuth, router_settings=None), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), route_type="aresponses", proxy_logging_obj=mock_proxy_logging, - llm_router=MagicMock(fallbacks=None), + llm_router=MagicMock(), general_settings={}, proxy_config=MagicMock(), ) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 92af65b2637..f689dd62df6 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -6365,247 +6365,170 @@ class TestPreCallWithFallbacksOnLocalRateLimit: call_type="acompletion", ) - @pytest.mark.asyncio - async def test_fallback_retries_from_pristine_request_data(self): - import threading + @staticmethod + def _v3_limiter_rig( + monkeypatch: pytest.MonkeyPatch, + user_api_key_dict: ProxyUserAPIKeyAuth, + fallbacks: list[dict[str, list[str]]], + ) -> tuple[ProxyLogging, litellm.Router, ProxyConfig, list[str]]: + """Real v3 limiter (the default ``parallel_request_limiter``) wired in through the + ``proxy_logging_obj`` seam, so ``common_processing_pre_call_logic`` runs for real: + ``add_litellm_data_to_request`` with a live OTel span, ``function_setup``, then the limiter.""" + from litellm.caching.caching import DualCache + from litellm.proxy import proxy_server + from litellm.proxy.hooks.parallel_request_limiter_v3 import _PROXY_MaxParallelRequestsHandler_v3 + from litellm.proxy.utils import InternalUsageCache - from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing - from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + monkeypatch.setattr(proxy_server, "prisma_client", None) + limiter = _PROXY_MaxParallelRequestsHandler_v3(internal_usage_cache=InternalUsageCache(DualCache())) + limiter_models: list[str] = [] - primary_model = "gpt-4" - fallback_model = "gpt-3.5-turbo" + async def run_limiter(**kwargs): + limiter_models.append(kwargs["data"]["model"]) + await limiter.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=DualCache(), + data=kwargs["data"], + call_type=kwargs["call_type"], + ) + return kwargs["data"] - processor = ProxyBaseLLMRequestProcessing( - data={ - "model": primary_model, - "messages": [{"role": "user", "content": "hi"}], - "metadata": {"tags": ["a"]}, - } + proxy_logging_obj = MagicMock(spec=ProxyLogging) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=run_limiter) + router = litellm.Router( + model_list=[ + {"model_name": group, "litellm_params": {"model": "openai/gpt-4.1-nano", "api_key": "fake"}} + for chain in fallbacks + for group in (*chain.keys(), *(m for models in chain.values() for m in models)) + ], + fallbacks=fallbacks, ) + return proxy_logging_obj, router, proxy_server.ProxyConfig(), limiter_models - metadata_at_entry = [] - - async def mock_pre_call_logic(**kwargs): - copy.deepcopy(processor.data["metadata"]) - metadata_at_entry.append(dict(processor.data["metadata"])) - processor.data["metadata"]["litellm_parent_otel_span"] = threading.RLock() - processor.data["litellm_logging_obj"] = object() - if processor.data.get("model") == primary_model: - raise ProxyRateLimitError( - detail="TPM limit exceeded for gpt-4", - headers={"retry-after": "30"}, - ) - return processor.data, MagicMock() - - mock_router = MagicMock() - mock_router.fallbacks = [{primary_model: [fallback_model]}] - - with patch.object( - processor, - "common_processing_pre_call_logic", - side_effect=mock_pre_call_logic, - ): - data, logging_obj = await processor._pre_call_with_fallbacks( - request=MagicMock(), - general_settings={}, - proxy_logging_obj=MagicMock(), - user_api_key_dict=MagicMock(router_settings=None), - version=None, - proxy_config=MagicMock(), - user_model=None, - user_temperature=None, - user_request_timeout=None, - user_max_tokens=None, - user_api_base=None, - model=primary_model, - route_type="acompletion", - llm_router=mock_router, - ) - - assert processor.data["model"] == fallback_model - assert metadata_at_entry[1] == {"tags": ["a"]} - - @pytest.mark.asyncio - async def test_exhausted_fallbacks_restore_pristine_request_data(self): - import threading - - from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing - from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError - - primary_model = "gpt-4" - original_data = { - "model": primary_model, - "messages": [{"role": "user", "content": "hi"}], - "metadata": {"tags": ["a"]}, - } - processor = ProxyBaseLLMRequestProcessing(data=copy.deepcopy(original_data)) - - async def mock_pre_call_logic(**kwargs): - processor.data["metadata"]["litellm_parent_otel_span"] = threading.RLock() - processor.data["litellm_logging_obj"] = object() - raise ProxyRateLimitError( - detail=f"TPM limit exceeded for {processor.data.get('model')}", - headers={"retry-after": "30"}, - ) - - mock_router = MagicMock() - mock_router.fallbacks = [{primary_model: ["gpt-3.5-turbo"]}] - - with patch.object( - processor, - "common_processing_pre_call_logic", - side_effect=mock_pre_call_logic, - ): - with pytest.raises(ProxyRateLimitError, match="gpt-4"): - await processor._pre_call_with_fallbacks( - request=MagicMock(), - general_settings={}, - proxy_logging_obj=MagicMock(), - user_api_key_dict=MagicMock(router_settings=None), - version=None, - proxy_config=MagicMock(), - user_model=None, - user_temperature=None, - user_request_timeout=None, - user_max_tokens=None, - user_api_base=None, - model=primary_model, - route_type="acompletion", - llm_router=mock_router, - ) - - assert processor.data == original_data - - @pytest.mark.asyncio - async def test_real_add_litellm_data_to_request_rerun_with_otel_span_falls_back(self): - from opentelemetry import trace + @staticmethod + def _otel_key(**limits) -> ProxyUserAPIKeyAuth: from opentelemetry.sdk.trace import TracerProvider - from litellm.proxy._types import UserAPIKeyAuth - from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing - from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError - from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request - from litellm.proxy.proxy_server import ProxyConfig + span = TracerProvider().get_tracer("test").start_span("proxy-request") + return ProxyUserAPIKeyAuth(api_key="hashed-key", parent_otel_span=span, **limits) - trace.set_tracer_provider(TracerProvider()) + @staticmethod + def _chat_request() -> Request: + return Request({"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": []}) - primary_model = "gpt-4" - fallback_model = "gpt-3.5-turbo" - - request_mock = MagicMock(spec=Request) - request_mock.url = MagicMock() - request_mock.url.path = "/v1/chat/completions" - request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" - request_mock.method = "POST" - request_mock.query_params = {} - request_mock.headers = {"Content-Type": "application/json"} - request_mock.client = MagicMock() - request_mock.client.host = "127.0.0.1" - - user_api_key_dict = UserAPIKeyAuth( - parent_otel_span=trace.get_tracer("x").start_span("s"), - api_key="hashed-key", - user_id="u1", - team_id="t1", - metadata={}, - team_metadata={}, - team_member_tpm_limit=1000, + async def _pre_call( + self, + data: dict, + user_api_key_dict: ProxyUserAPIKeyAuth, + rig: tuple[ProxyLogging, litellm.Router, ProxyConfig, list[str]], + ) -> tuple[ProxyBaseLLMRequestProcessing, tuple[dict, object]]: + proxy_logging_obj, router, proxy_config, _ = rig + processor = ProxyBaseLLMRequestProcessing(data=data) + result = await processor._pre_call_with_fallbacks( + request=self._chat_request(), + general_settings={}, + proxy_logging_obj=proxy_logging_obj, + user_api_key_dict=user_api_key_dict, + version=None, + proxy_config=proxy_config, + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=None, + route_type="acompletion", + llm_router=router, ) + return processor, result - processor = ProxyBaseLLMRequestProcessing( - data={ + @pytest.mark.asyncio + async def test_v3_limiter_with_otel_span_falls_back_from_client_request(self, monkeypatch: pytest.MonkeyPatch): + """Customer path: OTel on, per-key model RPM cap on the primary, a router fallback configured. + The first pass enriches ``data["metadata"]`` with the live span, then the limiter raises. The + fallback pass must start from the client's request again, so ``add_litellm_data_to_request`` + never deep-copies the span (the ``cannot pickle '_thread.RLock'`` 500).""" + primary_model = "gpt-4.1" + fallback_model = "gpt-4.1-mini" + key = self._otel_key(metadata={"model_rpm_limit": {primary_model: 1}}) + rig = self._v3_limiter_rig(monkeypatch, key, [{primary_model: [fallback_model]}]) + + def client_request() -> dict: + return { "model": primary_model, "messages": [{"role": "user", "content": "hi"}], - "metadata": {"tags": ["a"]}, + "metadata": {"tags": ["client-tag"]}, } - ) - async def real_add_litellm_data_pre_call(**kwargs): - await add_litellm_data_to_request( - data=processor.data, - request=request_mock, - user_api_key_dict=user_api_key_dict, - proxy_config=ProxyConfig(), + _, (first_data, _) = await self._pre_call(client_request(), key, rig) + processor, (data, logging_obj) = await self._pre_call(client_request(), key, rig) + + assert first_data["model"] == primary_model + assert data["model"] == fallback_model + assert processor.data is data + assert data["litellm_logging_obj"] is logging_obj + assert logging_obj.model == fallback_model + requester_metadata = data["metadata"]["requester_metadata"] + assert requester_metadata["tags"] == ["client-tag"] + assert "litellm_parent_otel_span" not in requester_metadata + assert "user_api_key_auth" not in requester_metadata + assert data["metadata"]["litellm_parent_otel_span"] is key.parent_otel_span + assert rig[3] == [primary_model, primary_model, fallback_model] + + @pytest.mark.asyncio + async def test_v3_limiter_with_otel_span_returns_429_when_fallbacks_exhausted( + self, monkeypatch: pytest.MonkeyPatch + ): + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + primary_model = "gpt-4.1" + fallback_model = "gpt-4.1-mini" + key = self._otel_key(rpm_limit=1) + rig = self._v3_limiter_rig(monkeypatch, key, [{primary_model: [fallback_model]}]) + request = {"model": primary_model, "messages": [{"role": "user", "content": "hi"}]} + + await self._pre_call(dict(request), key, rig) + processor = ProxyBaseLLMRequestProcessing(data=dict(request)) + with pytest.raises(ProxyRateLimitError) as exc_info: + await processor._pre_call_with_fallbacks( + request=self._chat_request(), general_settings={}, - version="test", - ) - if processor.data.get("model") == primary_model: - raise ProxyRateLimitError( - detail="TPM limit exceeded for gpt-4", - headers={"retry-after": "30"}, - ) - return processor.data, MagicMock() - - mock_router = MagicMock() - mock_router.fallbacks = [{primary_model: [fallback_model]}] - - with patch.object( - processor, - "common_processing_pre_call_logic", - side_effect=real_add_litellm_data_pre_call, - ): - data, logging_obj = await processor._pre_call_with_fallbacks( - request=request_mock, - general_settings={}, - proxy_logging_obj=MagicMock(), - user_api_key_dict=user_api_key_dict, + proxy_logging_obj=rig[0], + user_api_key_dict=key, version=None, - proxy_config=MagicMock(), + proxy_config=rig[2], user_model=None, user_temperature=None, user_request_timeout=None, user_max_tokens=None, user_api_base=None, - model=primary_model, + model=None, route_type="acompletion", - llm_router=mock_router, + llm_router=rig[1], ) - assert processor.data["model"] == fallback_model + assert rig[3] == [primary_model, primary_model, fallback_model] + assert exc_info.value.status_code == 429 + assert "Rate limit exceeded" in str(exc_info.value.detail) + assert exc_info.value.headers["retry-after"] + assert processor.data["model"] == primary_model + assert processor.data["litellm_logging_obj"].model == primary_model + assert processor.data["litellm_call_id"] @pytest.mark.asyncio - async def test_no_fallbacks_skips_snapshot(self): - from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing - from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + async def test_fallback_lookup_uses_alias_resolved_model_group(self, monkeypatch: pytest.MonkeyPatch): + primary_model = "gpt-4.1" + fallback_model = "gpt-4.1-mini" + monkeypatch.setattr(litellm, "model_alias_map", {"my-alias": primary_model}) + key = self._otel_key(metadata={"model_rpm_limit": {primary_model: 1}}) + rig = self._v3_limiter_rig(monkeypatch, key, [{primary_model: [fallback_model]}]) + request = {"model": "my-alias", "messages": [{"role": "user", "content": "hi"}]} - processor = ProxyBaseLLMRequestProcessing(data={"model": "gpt-4"}) + await self._pre_call(dict(request), key, rig) + _, (data, _) = await self._pre_call(dict(request), key, rig) - async def mock_pre_call_logic(**kwargs): - raise ProxyRateLimitError( - detail="TPM limit exceeded", - headers={"retry-after": "30"}, - ) - - mock_router = MagicMock() - mock_router.fallbacks = None - - with patch( # test-quality-ok: spying the snapshot seam is the only observable check that the no-fallback path skips it - "litellm.proxy.common_request_processing.independent_snapshot" - ) as snapshot_mock: - with patch.object( - processor, - "common_processing_pre_call_logic", - side_effect=mock_pre_call_logic, - ): - with pytest.raises(ProxyRateLimitError): - await processor._pre_call_with_fallbacks( - request=MagicMock(), - general_settings={}, - proxy_logging_obj=MagicMock(), - user_api_key_dict=MagicMock(router_settings=None), - version=None, - proxy_config=MagicMock(), - user_model=None, - user_temperature=None, - user_request_timeout=None, - user_max_tokens=None, - user_api_base=None, - model="gpt-4", - route_type="acompletion", - llm_router=mock_router, - ) - - snapshot_mock.assert_not_called() + assert data["model"] == fallback_model + assert rig[3] == [primary_model, primary_model, fallback_model] class _RecordingSuccessLogger(CustomLogger): From 9faaf7f4d436b558dbc4d1ab25a5801269629d6d Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 16 Sep 2026 07:15:13 +0000 Subject: [PATCH 5/7] refactor(proxy): assign the fallback model on the fresh snapshot instead of building a dict literal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7c9a296965c..d18fab1c9f0 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2113,7 +2113,8 @@ class ProxyBaseLLMRequestProcessing: for fallback_model in fallback_models: if fallback_model == original_model: continue - self.data = {**independent_snapshot(pristine), "model": fallback_model} + self.data = independent_snapshot(pristine) + self.data["model"] = fallback_model try: return await self.common_processing_pre_call_logic( request=request, From 391da46e2cdcef46491519ba2814b5d38c752285 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 16 Sep 2026 07:59:33 +0000 Subject: [PATCH 6/7] test: type the v3 limiter rig and otel key helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/test_common_request_processing.py | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index f689dd62df6..7d9a78fd981 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -44,6 +44,7 @@ from litellm.proxy.common_request_processing import ( create_response, ) from litellm.proxy.dd_span_tagger import DDSpanTagger +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import ProxyException from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth from litellm.proxy.utils import ProxyLogging @@ -6383,15 +6384,17 @@ class TestPreCallWithFallbacksOnLocalRateLimit: limiter = _PROXY_MaxParallelRequestsHandler_v3(internal_usage_cache=InternalUsageCache(DualCache())) limiter_models: list[str] = [] - async def run_limiter(**kwargs): - limiter_models.append(kwargs["data"]["model"]) + async def run_limiter( + user_api_key_dict: ProxyUserAPIKeyAuth, data: dict[str, object], call_type: str + ) -> dict[str, object]: + limiter_models.append(str(data["model"])) await limiter.async_pre_call_hook( user_api_key_dict=user_api_key_dict, cache=DualCache(), - data=kwargs["data"], - call_type=kwargs["call_type"], + data=data, + call_type=call_type, ) - return kwargs["data"] + return data proxy_logging_obj = MagicMock(spec=ProxyLogging) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=run_limiter) @@ -6406,11 +6409,18 @@ class TestPreCallWithFallbacksOnLocalRateLimit: return proxy_logging_obj, router, proxy_server.ProxyConfig(), limiter_models @staticmethod - def _otel_key(**limits) -> ProxyUserAPIKeyAuth: + def _otel_key( + rpm_limit: int | None = None, model_rpm_limit: dict[str, int] | None = None + ) -> ProxyUserAPIKeyAuth: from opentelemetry.sdk.trace import TracerProvider span = TracerProvider().get_tracer("test").start_span("proxy-request") - return ProxyUserAPIKeyAuth(api_key="hashed-key", parent_otel_span=span, **limits) + return ProxyUserAPIKeyAuth( + api_key="hashed-key", + parent_otel_span=span, + rpm_limit=rpm_limit, + metadata={"model_rpm_limit": model_rpm_limit} if model_rpm_limit else {}, + ) @staticmethod def _chat_request() -> Request: @@ -6418,10 +6428,10 @@ class TestPreCallWithFallbacksOnLocalRateLimit: async def _pre_call( self, - data: dict, + data: dict[str, object], user_api_key_dict: ProxyUserAPIKeyAuth, rig: tuple[ProxyLogging, litellm.Router, ProxyConfig, list[str]], - ) -> tuple[ProxyBaseLLMRequestProcessing, tuple[dict, object]]: + ) -> tuple[ProxyBaseLLMRequestProcessing, tuple[dict[str, object], LiteLLMLoggingObj]]: proxy_logging_obj, router, proxy_config, _ = rig processor = ProxyBaseLLMRequestProcessing(data=data) result = await processor._pre_call_with_fallbacks( @@ -6450,10 +6460,10 @@ class TestPreCallWithFallbacksOnLocalRateLimit: never deep-copies the span (the ``cannot pickle '_thread.RLock'`` 500).""" primary_model = "gpt-4.1" fallback_model = "gpt-4.1-mini" - key = self._otel_key(metadata={"model_rpm_limit": {primary_model: 1}}) + key = self._otel_key(model_rpm_limit={primary_model: 1}) rig = self._v3_limiter_rig(monkeypatch, key, [{primary_model: [fallback_model]}]) - def client_request() -> dict: + def client_request() -> dict[str, object]: return { "model": primary_model, "messages": [{"role": "user", "content": "hi"}], @@ -6520,7 +6530,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit: primary_model = "gpt-4.1" fallback_model = "gpt-4.1-mini" monkeypatch.setattr(litellm, "model_alias_map", {"my-alias": primary_model}) - key = self._otel_key(metadata={"model_rpm_limit": {primary_model: 1}}) + key = self._otel_key(model_rpm_limit={primary_model: 1}) rig = self._v3_limiter_rig(monkeypatch, key, [{primary_model: [fallback_model]}]) request = {"model": "my-alias", "messages": [{"role": "user", "content": "hi"}]} From 9974cf4bf817331053649fccc1f1b9e00d61d570 Mon Sep 17 00:00:00 2001 From: yucheng Date: Wed, 16 Sep 2026 08:17:58 +0000 Subject: [PATCH 7/7] fix(proxy): honor key-level disable_fallbacks after first pre-call pass Key metadata disable_fallbacks only lands on data during add_key_level_controls, so the local rate-limit fallback retry now rechecks it post pre-call. Also use a real UserAPIKeyAuth in the skip pre-call test since the path reads router_settings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 7 ++++- .../test_response_polling_pre_call_checks.py | 2 +- .../proxy/test_common_request_processing.py | 30 +++++++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index d18fab1c9f0..36b49ef064c 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2093,7 +2093,12 @@ class ProxyBaseLLMRequestProcessing: except ProxyRateLimitError as original_exc: rate_limited_data: Final = self.data original_model: Final = rate_limited_data.get("model") - if pristine is None or not configured_fallbacks or not isinstance(original_model, str): + if ( + pristine is None + or not configured_fallbacks + or rate_limited_data.get("disable_fallbacks") + or not isinstance(original_model, str) + ): raise fallback_models: Final = self._resolve_fallback_models( diff --git a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py index 459834d0fd2..9f1a228855e 100644 --- a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py +++ b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py @@ -87,7 +87,7 @@ class TestSkipPreCallLogic: await processor.base_process_llm_request( request=MagicMock(spec=Request), fastapi_response=MagicMock(spec=Response), - user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + user_api_key_dict=UserAPIKeyAuth(), route_type="aresponses", proxy_logging_obj=mock_proxy_logging, llm_router=MagicMock(), diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 7d9a78fd981..e9d74a46daa 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -6410,7 +6410,9 @@ class TestPreCallWithFallbacksOnLocalRateLimit: @staticmethod def _otel_key( - rpm_limit: int | None = None, model_rpm_limit: dict[str, int] | None = None + rpm_limit: int | None = None, + model_rpm_limit: dict[str, int] | None = None, + disable_fallbacks: bool = False, ) -> ProxyUserAPIKeyAuth: from opentelemetry.sdk.trace import TracerProvider @@ -6419,7 +6421,10 @@ class TestPreCallWithFallbacksOnLocalRateLimit: api_key="hashed-key", parent_otel_span=span, rpm_limit=rpm_limit, - metadata={"model_rpm_limit": model_rpm_limit} if model_rpm_limit else {}, + metadata={ + **({"model_rpm_limit": model_rpm_limit} if model_rpm_limit else {}), + **({"disable_fallbacks": True} if disable_fallbacks else {}), + }, ) @staticmethod @@ -6540,6 +6545,27 @@ class TestPreCallWithFallbacksOnLocalRateLimit: assert data["model"] == fallback_model assert rig[3] == [primary_model, primary_model, fallback_model] + @pytest.mark.asyncio + async def test_key_metadata_disable_fallbacks_returns_429_instead_of_retrying( + self, monkeypatch: pytest.MonkeyPatch + ): + """``disable_fallbacks`` set in key metadata only lands on ``data`` during the first + pre-call pass (``add_key_level_controls``), so it must be honored after that pass.""" + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + primary_model = "gpt-4.1" + fallback_model = "gpt-4.1-mini" + key = self._otel_key(model_rpm_limit={primary_model: 1}, disable_fallbacks=True) + rig = self._v3_limiter_rig(monkeypatch, key, [{primary_model: [fallback_model]}]) + request = {"model": primary_model, "messages": [{"role": "user", "content": "hi"}]} + + await self._pre_call(dict(request), key, rig) + with pytest.raises(ProxyRateLimitError) as exc_info: + await self._pre_call(dict(request), key, rig) + + assert exc_info.value.status_code == 429 + assert rig[3] == [primary_model, primary_model] + class _RecordingSuccessLogger(CustomLogger): def __init__(self):