From dff08dcb55b35ff9346445b0ba3a733692e249f1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:16:45 +0000 Subject: [PATCH 1/3] fix(proxy): retry rate-limit fallbacks from a pristine request snapshot Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 38 ++- .../proxy/test_common_request_processing.py | 240 ++++++++++++++++++ 2 files changed, 264 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index e3a2b892721..255bf3ebda2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -32,7 +32,11 @@ from litellm.constants import ( UNSAFE_PROXY_RESPONSE_HEADERS, ) from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket, is_expected_client_error +from litellm.litellm_core_utils.core_helpers import ( + get_or_create_metadata_bucket, + independent_snapshot, + is_expected_client_error, +) from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, @@ -2034,6 +2038,21 @@ class ProxyBaseLLMRequestProcessing: ) -> tuple[dict, LiteLLMLoggingObj]: from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + original_model: Final = self.data.get("model") + fallback_models: Final = ( + self._resolve_fallback_models( + model=original_model, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + ) + if original_model + and isinstance(original_model, str) + and llm_router + and not self.data.get("disable_fallbacks") + else None + ) + pristine: Final = independent_snapshot(self.data) if fallback_models else None + try: return await self.common_processing_pre_call_logic( request=request, @@ -2052,16 +2071,7 @@ class ProxyBaseLLMRequestProcessing: llm_router=llm_router, ) except ProxyRateLimitError as original_exc: - original_model: Final = self.data.get("model") - if not original_model or not llm_router or self.data.get("disable_fallbacks"): - raise - - fallback_models: Final = self._resolve_fallback_models( - model=original_model, - llm_router=llm_router, - user_api_key_dict=user_api_key_dict, - ) - if not fallback_models: + if not fallback_models or pristine is None: raise verbose_proxy_logger.info( @@ -2074,7 +2084,7 @@ class ProxyBaseLLMRequestProcessing: for fallback_model in fallback_models: if fallback_model == original_model: continue - self.data["model"] = fallback_model + self.data = {**independent_snapshot(pristine), "model": fallback_model} try: return await self.common_processing_pre_call_logic( request=request, @@ -2095,10 +2105,10 @@ class ProxyBaseLLMRequestProcessing: except ProxyRateLimitError: continue except BaseException: - self.data["model"] = original_model + self.data = pristine raise - self.data["model"] = original_model + self.data = pristine raise original_exc def _resolve_fallback_models( diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 50b26577e5c..89e0799ba16 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -6235,6 +6235,246 @@ class TestPreCallWithFallbacksOnLocalRateLimit: call_type="acompletion", ) + @pytest.mark.asyncio + async def test_fallback_retries_from_pristine_request_data(self): + import threading + + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + primary_model = "gpt-4" + fallback_model = "gpt-3.5-turbo" + + processor = ProxyBaseLLMRequestProcessing( + data={ + "model": primary_model, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"tags": ["a"]}, + } + ) + + metadata_at_entry = [] + + async def mock_pre_call_logic(**kwargs): + copy.deepcopy(processor.data["metadata"]) + metadata_at_entry.append(dict(processor.data["metadata"])) + processor.data["metadata"]["litellm_parent_otel_span"] = threading.RLock() + processor.data["litellm_logging_obj"] = object() + if processor.data.get("model") == primary_model: + raise ProxyRateLimitError( + detail="TPM limit exceeded for gpt-4", + headers={"retry-after": "30"}, + ) + return processor.data, MagicMock() + + mock_router = MagicMock() + mock_router.fallbacks = [{primary_model: [fallback_model]}] + + with patch.object( + processor, + "common_processing_pre_call_logic", + side_effect=mock_pre_call_logic, + ): + data, logging_obj = await processor._pre_call_with_fallbacks( + request=MagicMock(), + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=MagicMock(router_settings=None), + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=primary_model, + route_type="acompletion", + llm_router=mock_router, + ) + + assert processor.data["model"] == fallback_model + assert metadata_at_entry[1] == {"tags": ["a"]} + + @pytest.mark.asyncio + async def test_exhausted_fallbacks_restore_pristine_request_data(self): + import threading + + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + primary_model = "gpt-4" + original_data = { + "model": primary_model, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"tags": ["a"]}, + } + processor = ProxyBaseLLMRequestProcessing(data=copy.deepcopy(original_data)) + + async def mock_pre_call_logic(**kwargs): + processor.data["metadata"]["litellm_parent_otel_span"] = threading.RLock() + processor.data["litellm_logging_obj"] = object() + raise ProxyRateLimitError( + detail=f"TPM limit exceeded for {processor.data.get('model')}", + headers={"retry-after": "30"}, + ) + + mock_router = MagicMock() + mock_router.fallbacks = [{primary_model: ["gpt-3.5-turbo"]}] + + with patch.object( + processor, + "common_processing_pre_call_logic", + side_effect=mock_pre_call_logic, + ): + with pytest.raises(ProxyRateLimitError, match="gpt-4"): + await processor._pre_call_with_fallbacks( + request=MagicMock(), + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=MagicMock(router_settings=None), + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=primary_model, + route_type="acompletion", + llm_router=mock_router, + ) + + assert processor.data == original_data + + @pytest.mark.asyncio + async def test_real_add_litellm_data_to_request_rerun_with_otel_span_falls_back(self): + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.proxy_server import ProxyConfig + + trace.set_tracer_provider(TracerProvider()) + + primary_model = "gpt-4" + fallback_model = "gpt-3.5-turbo" + + request_mock = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.path = "/v1/chat/completions" + request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + user_api_key_dict = UserAPIKeyAuth( + parent_otel_span=trace.get_tracer("x").start_span("s"), + api_key="hashed-key", + user_id="u1", + team_id="t1", + metadata={}, + team_metadata={}, + team_member_tpm_limit=1000, + ) + + processor = ProxyBaseLLMRequestProcessing( + data={ + "model": primary_model, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"tags": ["a"]}, + } + ) + + async def real_add_litellm_data_pre_call(**kwargs): + await add_litellm_data_to_request( + data=processor.data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=ProxyConfig(), + general_settings={}, + version="test", + ) + if processor.data.get("model") == primary_model: + raise ProxyRateLimitError( + detail="TPM limit exceeded for gpt-4", + headers={"retry-after": "30"}, + ) + return processor.data, MagicMock() + + mock_router = MagicMock() + mock_router.fallbacks = [{primary_model: [fallback_model]}] + + with patch.object( + processor, + "common_processing_pre_call_logic", + side_effect=real_add_litellm_data_pre_call, + ): + data, logging_obj = await processor._pre_call_with_fallbacks( + request=request_mock, + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=user_api_key_dict, + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=primary_model, + route_type="acompletion", + llm_router=mock_router, + ) + + assert processor.data["model"] == fallback_model + + @pytest.mark.asyncio + async def test_no_fallbacks_skips_snapshot(self): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + processor = ProxyBaseLLMRequestProcessing(data={"model": "gpt-4"}) + + async def mock_pre_call_logic(**kwargs): + raise ProxyRateLimitError( + detail="TPM limit exceeded", + headers={"retry-after": "30"}, + ) + + mock_router = MagicMock() + mock_router.fallbacks = None + + with patch("litellm.proxy.common_request_processing.independent_snapshot") as snapshot_mock: + with patch.object( + processor, + "common_processing_pre_call_logic", + side_effect=mock_pre_call_logic, + ): + with pytest.raises(ProxyRateLimitError): + await processor._pre_call_with_fallbacks( + request=MagicMock(), + general_settings={}, + proxy_logging_obj=MagicMock(), + user_api_key_dict=MagicMock(router_settings=None), + version=None, + proxy_config=MagicMock(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model="gpt-4", + route_type="acompletion", + llm_router=mock_router, + ) + + snapshot_mock.assert_not_called() + class _RecordingSuccessLogger(CustomLogger): def __init__(self): From c064e576ee31ed05b067462412c6c234d364c8fd Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:29:20 +0000 Subject: [PATCH 2/3] fix(proxy): tolerate missing router_settings and non-list fallbacks in fallback resolution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 4 ++-- tests/test_litellm/proxy/test_common_request_processing.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 255bf3ebda2..9927320d795 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2121,14 +2121,14 @@ class ProxyBaseLLMRequestProcessing: fallbacks = None - key_router_settings: Final = user_api_key_dict.router_settings + key_router_settings: Final = getattr(user_api_key_dict, "router_settings", None) if isinstance(key_router_settings, dict) and "fallbacks" in key_router_settings: fallbacks = key_router_settings["fallbacks"] if fallbacks is None: fallbacks = llm_router.fallbacks - if not fallbacks: + if not isinstance(fallbacks, list) or not fallbacks: return None fallback_model_group, generic_fallback_idx = get_fallback_model_group( diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 89e0799ba16..4beb73b3a81 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -6449,7 +6449,9 @@ class TestPreCallWithFallbacksOnLocalRateLimit: mock_router = MagicMock() mock_router.fallbacks = None - with patch("litellm.proxy.common_request_processing.independent_snapshot") as snapshot_mock: + with patch( # test-quality-ok: spying the snapshot seam is the only observable check that the no-fallback path skips it + "litellm.proxy.common_request_processing.independent_snapshot" + ) as snapshot_mock: with patch.object( processor, "common_processing_pre_call_logic", From 431dcce6a72ddcb6ba73c56848e1b5e0477b7699 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:46:44 +0000 Subject: [PATCH 3/3] test(proxy): give pre-call mocks real router_settings and fallbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 4 ++-- .../test_response_polling_pre_call_checks.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 9927320d795..255bf3ebda2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2121,14 +2121,14 @@ class ProxyBaseLLMRequestProcessing: fallbacks = None - key_router_settings: Final = getattr(user_api_key_dict, "router_settings", None) + key_router_settings: Final = user_api_key_dict.router_settings if isinstance(key_router_settings, dict) and "fallbacks" in key_router_settings: fallbacks = key_router_settings["fallbacks"] if fallbacks is None: fallbacks = llm_router.fallbacks - if not isinstance(fallbacks, list) or not fallbacks: + if not fallbacks: return None fallback_model_group, generic_fallback_idx = get_fallback_model_group( diff --git a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py index 459834d0fd2..38f087f51ca 100644 --- a/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py +++ b/tests/proxy_unit_tests/test_response_polling_pre_call_checks.py @@ -48,10 +48,10 @@ class TestSkipPreCallLogic: await processor.base_process_llm_request( request=MagicMock(spec=Request), fastapi_response=MagicMock(spec=Response), - user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth, router_settings=None), route_type="aresponses", proxy_logging_obj=mock_proxy_logging, - llm_router=MagicMock(), + llm_router=MagicMock(fallbacks=None), general_settings={}, proxy_config=MagicMock(), skip_pre_call_logic=True, @@ -87,10 +87,10 @@ class TestSkipPreCallLogic: await processor.base_process_llm_request( request=MagicMock(spec=Request), fastapi_response=MagicMock(spec=Response), - user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), + user_api_key_dict=MagicMock(spec=UserAPIKeyAuth, router_settings=None), route_type="aresponses", proxy_logging_obj=mock_proxy_logging, - llm_router=MagicMock(), + llm_router=MagicMock(fallbacks=None), general_settings={}, proxy_config=MagicMock(), )