diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index bef07c68e9e..6e6d4c253e9 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -640,8 +640,13 @@ def cast_exception_status_to_int(exception_status: str | int) -> int: return exception_status -def is_caller_timeout_408(model_call_details: Mapping[str, object], exception_status: str | int) -> bool: - """A 408 that arrives before the caller-set timeout could have fired came from the provider.""" +def is_caller_timeout_408( + model_call_details: Mapping[str, object], exception_status: str | int, ended: datetime | None = None +) -> bool: + """A 408 that arrives before the caller-set timeout could have fired came from the provider. + + ``ended`` overrides ``model_call_details["end_time"]`` for callers that run before the + failure logger has stamped the current API call's end time.""" if cast_exception_status_to_int(exception_status) != 408: return False litellm_params: Final = model_call_details.get("litellm_params") @@ -649,7 +654,7 @@ def is_caller_timeout_408(model_call_details: Mapping[str, object], exception_st return False timeout: Final = litellm_params.get("timeout") started: Final = model_call_details.get("api_call_start_time") or model_call_details.get("start_time") - ended: Final = model_call_details.get("end_time") - if not isinstance(timeout, (int, float)) or not isinstance(started, datetime) or not isinstance(ended, datetime): + finished: Final = ended if ended is not None else model_call_details.get("end_time") + if not isinstance(timeout, (int, float)) or not isinstance(started, datetime) or not isinstance(finished, datetime): return False - return (ended - started).total_seconds() >= timeout + return (finished - started).total_seconds() >= timeout diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index eeea9b9faf8..94164d0ea0c 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -2,6 +2,7 @@ import hashlib import json from collections.abc import Mapping, Sequence from dataclasses import dataclass +from datetime import datetime from enum import Enum from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -84,7 +85,11 @@ def _trigger_cooldown_for_failed_deployment( # timeout, which litellm.Timeout reports as status 408 regardless of the deployment's # actual health. Left unguarded, a caller could force a 408 on every deployment in # the fallback chain from a single request with a near-zero timeout. - if is_caller_timeout_408(model_call_details, exception_status): + if is_caller_timeout_408( + model_call_details, + exception_status, + ended=datetime.now(), # noqa: DTZ005 # naive to match the logging pipeline's api_call_start_time + ): verbose_router_logger.debug( "Not triggering cooldown for fallback deployment: a caller-supplied " "x-litellm-timeout caused this 408, not deployment health." diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py index 783f8da31e9..9318f306c89 100644 --- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py +++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py @@ -956,7 +956,11 @@ class TestTriggerCooldownForFailedDeployment: """The proxy's x-litellm-timeout header lets a caller set an arbitrarily short timeout, which litellm.Timeout reports as status 408 regardless of the deployment's actual health. Without this guard, a caller could force a 408 on - every deployment in the fallback chain from a single request.""" + every deployment in the fallback chain from a single request. + + The failure logger never stamps end_time for a fallback hop (has_logged_async_failure + is already set), so model_call_details still carries the previous hop's end_time, which + predates this hop's api_call_start_time. The guard must not trust it.""" mock_router = MagicMock() mock_router.cooldown_time = 60.0 mock_router.get_model_info.return_value = None @@ -977,7 +981,7 @@ class TestTriggerCooldownForFailedDeployment: model_call_details={ "litellm_params": {"client_side_timeout": True, "timeout": 0.5}, "api_call_start_time": datetime.now() - timedelta(seconds=1), - "end_time": datetime.now(), + "end_time": datetime.now() - timedelta(seconds=5), }, )