From 0be6e9c568dab3b05400370183cdf9eb3201d7ea Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 23:14:04 +0000 Subject: [PATCH] fix(router): run cooldown callback on every failed deployment attempt Router retries/fallbacks reuse the same Logging object, so the sync_failure dedup in failure_handler skipped deployment_callback_on_failure after the first failure and left later failed deployments out of cooldown. Run the router cooldown/usage callbacks on every failed attempt while keeping observability failure callbacks deduplicated once per request. Fixes #32574 --- litellm/litellm_core_utils/litellm_logging.py | 24 ++++++-- .../test_litellm_logging.py | 61 +++++++++++++++++++ 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 936d79b22d6..ecb2b02eebd 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2753,8 +2753,7 @@ class Logging(LiteLLMLoggingBaseClass): def failure_handler(self, exception, traceback_exception, start_time=None, end_time=None): verbose_logger.debug(f"Logging Details LiteLLM-Failure Call: {litellm.failure_callback}") - if not self.should_run_logging(event_type="sync_failure"): # prevent double logging - return + should_log_failure = self.should_run_logging(event_type="sync_failure") # prevent double logging litellm_params = self.model_call_details.get("litellm_params", {}) is_sync_request = self._is_sync_litellm_request(litellm_params) @@ -2776,9 +2775,12 @@ class Logging(LiteLLMLoggingBaseClass): model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), result=result, ) - self.has_run_logging(event_type="sync_failure") + if should_log_failure: + self.has_run_logging(event_type="sync_failure") for callback in callbacks: try: + if not should_log_failure and not self._is_router_cooldown_callback(callback): + continue should_run = self.should_run_callback( callback=callback, litellm_params=litellm_params, @@ -2925,8 +2927,7 @@ class Logging(LiteLLMLoggingBaseClass): Implementing async callbacks, to handle asyncio event loop issues when custom integrations need to use async functions. """ await self.special_failure_handlers(exception=exception) - if not self.should_run_logging(event_type="async_failure"): # prevent double logging - return + should_log_failure = self.should_run_logging(event_type="async_failure") # prevent double logging start_time, end_time = self._failure_handler_helper_fn( exception=exception, traceback_exception=traceback_exception, @@ -2941,10 +2942,13 @@ class Logging(LiteLLMLoggingBaseClass): result = None # result sent to all loggers, init this to None incase it's not created - self.has_run_logging(event_type="async_failure") + if should_log_failure: + self.has_run_logging(event_type="async_failure") for callback in callbacks: try: litellm_params = self.model_call_details.get("litellm_params", {}) + if not should_log_failure and not self._is_router_cooldown_callback(callback): + continue should_run = self.should_run_callback( callback=callback, litellm_params=litellm_params, @@ -3107,6 +3111,14 @@ class Logging(LiteLLMLoggingBaseClass): return cb.__class__.__name__ return str(cb) + def _is_router_cooldown_callback(self, cb) -> bool: + if not callable(cb): + return False + return self._get_callback_name(cb) in ( + "deployment_callback_on_failure", + "async_deployment_callback_on_failure", + ) + def _is_internal_litellm_proxy_callback(self, cb) -> bool: """Helper to check if a callback is internal""" INTERNAL_PREFIXES = [ diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 0523ed7ecb1..d30d7d39a25 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -2871,6 +2871,67 @@ def test_failure_handler_runs_sync_callbacks_for_non_pass_through_requests( dummy_logger.log_failure_event.assert_called_once() +def test_failure_handler_runs_router_cooldown_callback_on_every_attempt(logging_obj): + """Regression for #32574: router retries/fallbacks reuse the same Logging object, so + sync_failure dedup used to skip deployment_callback_on_failure after the first failure + and leave later failed deployments out of cooldown. The cooldown callback must fire on + every failed attempt while observability callbacks stay deduplicated (once per request). + """ + cooldown_calls = [] + observability_calls = [] + + def deployment_callback_on_failure(kwargs, completion_response, start_time, end_time): + cooldown_calls.append(kwargs.get("exception")) + + def observability_cb(kwargs, completion_response, start_time, end_time): + observability_calls.append(kwargs.get("exception")) + + logging_obj.stream = False + logging_obj.model_call_details["litellm_params"] = {} + logging_obj.litellm_params = {} + + with patch.object( + logging_obj, + "get_combined_callback_list", + return_value=[deployment_callback_on_failure, observability_cb], + ): + for _ in range(2): + logging_obj.failure_handler(exception=Exception("429"), traceback_exception="") + + assert len(cooldown_calls) == 2 + assert len(observability_calls) == 1 + + +@pytest.mark.asyncio +async def test_async_failure_handler_runs_router_cooldown_callback_on_every_attempt(logging_obj): + """Async counterpart of #32574: async_deployment_callback_on_failure must run on every + failed attempt even though observability failure callbacks are deduplicated. + """ + cooldown_calls = [] + observability_calls = [] + + async def async_deployment_callback_on_failure(kwargs, completion_response, start_time, end_time): + cooldown_calls.append(kwargs.get("exception")) + + async def observability_cb(kwargs, completion_response, start_time, end_time): + observability_calls.append(kwargs.get("exception")) + + logging_obj.stream = False + logging_obj.model_call_details["litellm_params"] = {} + logging_obj.litellm_params = {} + + with patch.object( + logging_obj, + "get_combined_callback_list", + return_value=[async_deployment_callback_on_failure, observability_cb], + ): + for _ in range(2): + await logging_obj.async_failure_handler(exception=Exception("429"), traceback_exception="") + + assert len(cooldown_calls) == 2 + assert len(observability_calls) == 1 + + def test_merge_hidden_params_from_response_into_metadata_populates_metadata(): """Streaming completion path should mirror non-stream: metadata.hidden_params from response.""" from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj