From a9c422735fb8eb2f3e54510fac4796516e29107e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 15 Sep 2026 05:39:23 -0700 Subject: [PATCH] fix(router): stop counting caller-set timeout 408s toward deployment cooldown A 408 produced by a timeout the caller set (a timeout body field or an x-litellm-timeout header, which the proxy marks as client_side_timeout) says nothing about the deployment's health, yet the router's primary failure callback counted it toward allowed_fails and cooled the deployment down. The fallback path already skipped it. The marker never reached that callback because get_litellm_params drops kwargs outside OPTIONAL_KWARGS_KEYS, so it is listed there now, and deployment_callback_on_failure returns before the failure counter when is_caller_timeout_408 holds. A 408 from a timeout the deployment or the provider set still counts and still cools the deployment down. --- .../litellm_core_utils/get_litellm_params.py | 1 + litellm/router.py | 8 ++ litellm/router_utils/cooldown_handlers.py | 4 + .../router_utils/fallback_event_handlers.py | 3 +- tests/test_litellm/test_router.py | 82 +++++++++++++++++++ 5 files changed, 97 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index edd2e88f95c..49fc9abc525 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -41,6 +41,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "azure_password", "azure_scope", "timeout", + "client_side_timeout", "gcs_bucket_name", "bucket_name", "vertex_credentials", diff --git a/litellm/router.py b/litellm/router.py index ef89d611075..fc698ecb4b9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -167,6 +167,7 @@ from litellm.router_utils.cooldown_handlers import ( _get_cooldown_deployments, _set_cooldown_deployments, is_advisor_orchestration_failure, + is_caller_timeout_408, ) from litellm.router_utils.fallback_event_handlers import ( AttemptedFallbackTargets, @@ -8297,6 +8298,13 @@ class Router: litellm_params: Final = kwargs.get("litellm_params", {}) _model_info: Final = litellm_params.get("model_info", {}) + if is_caller_timeout_408(litellm_params.get("client_side_timeout"), exception_status): + verbose_router_logger.debug( + "Router: Exiting 'deployment_callback_on_failure' without cooldown. " + "A timeout the caller set caused this 408, not the deployment's health." + ) + return False + exception_headers: Final = litellm.litellm_core_utils.exception_mapping_utils._get_response_headers( original_exception=exception ) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 027f0a9ca05..e21567684eb 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -637,3 +637,7 @@ def cast_exception_status_to_int(exception_status: str | int) -> int: ) exception_status = 500 return exception_status + + +def is_caller_timeout_408(client_side_timeout: object, exception_status: str | int) -> bool: + return bool(client_side_timeout) and cast_exception_status_to_int(exception_status) == 408 diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 7fda5d96fb0..527a6b484e0 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -20,6 +20,7 @@ from litellm.router_utils.cooldown_handlers import ( _set_cooldown_deployments, # pyright: ignore[reportPrivateUsage] - shared helper, used across router_utils cast_exception_status_to_int, is_advisor_orchestration_failure, + is_caller_timeout_408, ) from litellm.router_utils.router_callbacks.track_deployment_metrics import ( increment_deployment_failures_for_current_minute, @@ -80,7 +81,7 @@ def _trigger_cooldown_for_failed_deployment( # timeout, which litellm.Timeout reports as status 408 regardless of the deployment's # actual health. Left unguarded, a caller could force a 408 on every deployment in # the fallback chain from a single request with a near-zero timeout. - if kwargs.get("client_side_timeout") and cast_exception_status_to_int(exception_status) == 408: + if is_caller_timeout_408(kwargs.get("client_side_timeout"), exception_status): verbose_router_logger.debug( "Not triggering cooldown for fallback deployment: a caller-supplied " "x-litellm-timeout caused this 408, not deployment health." diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c46a080976c..53eb91e6c63 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8520,6 +8520,88 @@ class TestAdvisorSubCallCooldown: assert "dep-1" not in self._cooled_down_ids(router) +class TestCallerTimeoutCooldown: + """A timeout the caller set (the proxy's `timeout` body field or x-litellm-timeout + header) comes back as a 408 whatever the deployment's health, so it must neither + count toward allowed_fails nor bench the deployment. A 408 without that marker is + the provider's and keeps cooling the deployment down.""" + + def _router(self): + return litellm.Router( + model_list=[ + { + "model_name": "slow-model", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "sk-fake"}, + "model_info": {"id": "dep-1"}, + } + ], + allowed_fails=0, + cooldown_time=120, + num_retries=0, + ) + + def _kwargs(self, marker): + exception = litellm.Timeout(message="Request timed out", model="gpt-5.6", llm_provider="openai") + return { + "exception": exception, + "litellm_params": {"model_info": {"id": "dep-1"}, "metadata": {}, **marker}, + } + + def _fail_count(self, router): + from litellm.router_utils.router_callbacks.track_deployment_metrics import ( + get_deployment_failures_for_current_minute, + ) + + return get_deployment_failures_for_current_minute(litellm_router_instance=router, deployment_id="dep-1") + + def _cooled_down_ids(self, router): + active = router.cooldown_cache.get_active_cooldowns(model_ids=["dep-1"], parent_otel_span=None) + return [entry[0] for entry in active] + + @pytest.mark.asyncio + async def test_caller_timeout_408_leaves_failure_counter_and_cooldown_untouched(self): + router = self._router() + now = datetime.now() + assert router.deployment_callback_on_failure(self._kwargs({"client_side_timeout": True}), None, now, now) is False + assert self._fail_count(router) == 0 + assert self._cooled_down_ids(router) == [] + + @pytest.mark.asyncio + async def test_provider_timeout_408_still_counts_and_cools_down(self): + router = self._router() + now = datetime.now() + assert router.deployment_callback_on_failure(self._kwargs({}), None, now, now) is True + assert self._fail_count(router) == 1 + assert self._cooled_down_ids(router) == ["dep-1"] + + @pytest.mark.asyncio + async def test_caller_timeout_marker_reaches_failure_callback_end_to_end(self): + router = self._router() + seen = [] + recorded = threading.Event() + + def record(kwargs, completion_response, start_time, end_time): + seen.append(kwargs) + recorded.set() + + litellm.failure_callback.append(record) + try: + with pytest.raises(litellm.Timeout): + await router.acompletion( + model="slow-model", + messages=[{"role": "user", "content": "hello"}], + mock_timeout=True, + timeout=0.001, + client_side_timeout=True, + ) + assert await asyncio.to_thread(recorded.wait, 5) + finally: + litellm.failure_callback.remove(record) + assert seen[0]["litellm_params"]["client_side_timeout"] is True + assert self._fail_count(router) == 0 + assert self._cooled_down_ids(router) == [] + + def test_stream_chunks_have_generated_content_detects_text_and_non_text(): from litellm.router import _stream_chunks_have_generated_content from litellm.types.utils import (