diff --git a/litellm/router.py b/litellm/router.py index 3f3daaf2eae..05c03dd69f7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -179,6 +179,7 @@ from litellm.router_utils.cooldown_handlers import ( _get_cooldown_deployments, _set_cooldown_deployments, is_advisor_orchestration_failure, + is_background_response_cost_poll_failure, is_caller_timeout_408, ) from litellm.router_utils.fallback_event_handlers import ( @@ -8137,12 +8138,19 @@ class Router: ) return False - exception_status: Final = getattr(exception, "status_code", "") - # Cache litellm_params to avoid repeated dict lookups litellm_params: Final = kwargs.get("litellm_params", {}) _model_info: Final = litellm_params.get("model_info", {}) + if is_background_response_cost_poll_failure(litellm_params): + verbose_router_logger.debug( + "Router: Exiting 'deployment_callback_on_failure' without cooldown. " + "Failure came from the background response cost poll, not the deployment's health." + ) + return False + + exception_status: Final = getattr(exception, "status_code", "") + if is_caller_timeout_408(kwargs, exception_status): verbose_router_logger.debug( "Router: Exiting 'deployment_callback_on_failure' without cooldown. " diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 6e6d4c253e9..cda1cca497c 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -20,9 +20,11 @@ from litellm.constants import ( DEFAULT_COOLDOWN_TIME_SECONDS, DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS, DEFAULT_FAILURE_THRESHOLD_PERCENT, + INTERNAL_CALL_ORIGIN_METADATA_KEY, SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD, ) from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback +from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN from .router_callbacks.track_deployment_metrics import ( get_deployment_failures_for_current_minute, @@ -62,6 +64,18 @@ def is_advisor_orchestration_failure(exception: BaseException | None) -> bool: return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False)) +def is_background_response_cost_poll_failure(litellm_params: Mapping[str, object]) -> bool: + """Whether the failed call was the enterprise cost poller reading back a stored background response. + + A provider 404 there means the provider dropped the stored object, not that the deployment is unhealthy. + """ + return any( + isinstance(candidate, Mapping) + and candidate.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN + for candidate in (litellm_params.get("metadata"), litellm_params.get("litellm_metadata")) + ) + + _EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = ( # ContentPolicyViolationError subclasses BadRequestError, so it must be checked first. (litellm.ContentPolicyViolationError, "ContentPolicyViolationErrorAllowedFails"), diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c5ae5d4b151..e6037bebabe 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8779,6 +8779,104 @@ class TestAdvisorSubCallCooldown: assert "dep-1" not in self._cooled_down_ids(router) +class TestBackgroundResponseCostPollCooldown: + def _router(self): + return litellm.Router( + model_list=[ + { + "model_name": "gpt-4.1", + "litellm_params": {"model": "openai/gpt-4.1"}, + "model_info": {"id": "dep-1"}, + } + ], + ) + + def _cooled_down_ids(self, router): + active = router.cooldown_cache.get_active_cooldowns(model_ids=["dep-1"], parent_otel_span=None) + return [entry[0] for entry in active] + + def _not_found(self): + return litellm.NotFoundError( + message="Response with id 'resp_gone' not found.", llm_provider="openai", model="gpt-4.1" + ) + + def _deployment_callback_on_failure(self, router, kwargs): + import asyncio + from datetime import datetime + + async def callback(): + now = datetime.now() + return router.deployment_callback_on_failure(kwargs, None, now, now) + + return asyncio.run(callback()) + + def test_untagged_not_found_cools_down_deployment(self): + router = self._router() + assert ( + self._deployment_callback_on_failure( + router, + { + "exception": self._not_found(), + "litellm_params": {"model_info": {"id": "dep-1"}, "metadata": {}}, + }, + ) + is True + ) + assert "dep-1" in self._cooled_down_ids(router) + + def test_cost_poll_not_found_does_not_cool_down_deployment(self): + from datetime import datetime + + from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY + from litellm.router_utils.router_callbacks.track_deployment_metrics import ( + get_deployment_failures_for_current_minute, + ) + from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN + + router = self._router() + now = datetime.now() + assert ( + router.deployment_callback_on_failure( + { + "exception": self._not_found(), + "litellm_params": { + "model_info": {"id": "dep-1"}, + "litellm_metadata": { + INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN + }, + }, + }, + None, + now, + now, + ) + is False + ) + assert self._cooled_down_ids(router) == [] + value = get_deployment_failures_for_current_minute(litellm_router_instance=router, deployment_id="dep-1") + assert not value + + def test_other_internal_origin_not_found_still_cools_down_deployment(self): + from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY + from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN + + router = self._router() + assert ( + self._deployment_callback_on_failure( + router, + { + "exception": self._not_found(), + "litellm_params": { + "model_info": {"id": "dep-1"}, + "litellm_metadata": {INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN}, + }, + }, + ) + is True + ) + assert "dep-1" in self._cooled_down_ids(router) + + class TestCallerTimeoutCooldown: """A timeout the caller set (the proxy's `timeout` body field or x-litellm-timeout header) comes back as a 408 whatever the deployment's health, so it must neither