fix(router): cooldown unreachable deployments for failover

litellm/router_utils/cooldown_handlers.py

@@ -54,13 +54,9 @@ def _is_cooldown_required(
         bool: True if a cooldown is required, False otherwise.
     """
     try:
-        ignored_strings = ["APIConnectionError"]
-        if (
-            exception_str is not None
-        ):  # don't cooldown on litellm api connection errors errors
-            for ignored_string in ignored_strings:
-                if ignored_string in exception_str:
-                    return False
+        # Connection errors mean the selected deployment was unreachable. Let
+        # the normal status-code and allowed-fails logic decide whether to
+        # cool it down so retries can fail over to another deployment.
 
         if isinstance(exception_status, str):
             if len(exception_status) == 0:

tests/local_testing/test_router.py

@@ -1915,38 +1915,21 @@ def test_router_context_window_pre_call_check(model, base_model, llm_provider):
 def test_router_cooldown_api_connection_error():
     from litellm.router_utils.cooldown_handlers import _is_cooldown_required
 
-    try:
-        _ = litellm.completion(
-            model="vertex_ai/gemini-1.5-pro",
-            messages=[{"role": "admin", "content": "Fail on this!"}],
-        )
-    except litellm.APIConnectionError as e:
-        assert (
-            _is_cooldown_required(
-                litellm_router_instance=Router(),
-                model_id="",
-                exception_status=e.code,
-                exception_str=str(e),
-            )
-            is False
-        )
-
-    router = Router(
-        model_list=[
-            {
-                "model_name": "gemini-1.5-pro",
-                "litellm_params": {"model": "vertex_ai/gemini-1.5-pro"},
-            }
-        ]
+    error = litellm.APIConnectionError(
+        message="Cannot connect to host calormen:11434",
+        llm_provider="ollama",
+        model="ollama_chat/gemma3:12b",
     )
 
-    try:
-        router.completion(
-            model="gemini-1.5-pro",
-            messages=[{"role": "admin", "content": "Fail on this!"}],
+    assert (
+        _is_cooldown_required(
+            litellm_router_instance=Router(),
+            model_id="dead-deployment",
+            exception_status=error.status_code,
+            exception_str=str(error),
         )
-    except litellm.APIConnectionError:
-        pass
+        is True
+    )
 
 
 def test_router_correctly_reraise_error():

tests/router_unit_tests/test_router_cooldown_utils.py

@@ -18,6 +18,7 @@ from litellm.router_utils.cooldown_handlers import (
     _should_run_cooldown_logic,
     _should_cooldown_deployment,
     cast_exception_status_to_int,
+    _get_cooldown_deployments,
     _is_cooldown_required,
 )
 from litellm.router_utils.router_callbacks.track_deployment_metrics import (
@@ -416,6 +417,79 @@ def test_is_cooldown_required_empty_string_exception_status(testing_litellm_rout
     ), "Should not require cooldown when exception_status is empty string"
 
 
+def test_api_connection_error_requires_cooldown(testing_litellm_router):
+    error = litellm.APIConnectionError(
+        message="Cannot connect to host calormen:11434",
+        llm_provider="ollama",
+        model="ollama_chat/gemma3:12b",
+    )
+
+    assert (
+        _is_cooldown_required(
+            litellm_router_instance=testing_litellm_router,
+            model_id="dead-deployment",
+            exception_status=error.status_code,
+            exception_str=str(error),
+        )
+        is True
+    )
+
+
+@pytest.mark.asyncio
+async def test_api_connection_error_cooldown_filters_failed_deployment():
+    router = Router(
+        model_list=[
+            {
+                "model_name": "local-model",
+                "litellm_params": {
+                    "model": "ollama_chat/gemma3:12b",
+                    "api_base": "http://dead-host:11434",
+                },
+                "model_info": {"id": "dead-deployment"},
+            },
+            {
+                "model_name": "local-model",
+                "litellm_params": {
+                    "model": "ollama_chat/gemma3:12b",
+                    "api_base": "http://healthy-host:11434",
+                },
+                "model_info": {"id": "healthy-deployment"},
+            },
+        ],
+        routing_strategy="simple-shuffle",
+        allowed_fails=0,
+        cooldown_time=60,
+    )
+    error = litellm.APIConnectionError(
+        message="Cannot connect to host dead-host:11434",
+        llm_provider="ollama",
+        model="ollama_chat/gemma3:12b",
+    )
+
+    did_cooldown = router.deployment_callback_on_failure(
+        kwargs={
+            "exception": error,
+            "litellm_params": {
+                "metadata": {"model_group": "local-model"},
+                "model_info": {"id": "dead-deployment"},
+            },
+        },
+        completion_response=None,
+        start_time=time.time(),
+        end_time=time.time(),
+    )
+
+    assert did_cooldown is True
+    assert "dead-deployment" in _get_cooldown_deployments(
+        litellm_router_instance=router,
+        parent_otel_span=None,
+    )
+
+    selected = router.get_available_deployment(model="local-model")
+    assert selected is not None
+    assert selected["model_info"]["id"] == "healthy-deployment"
+
+
 def test_should_cooldown_deployment_minimum_request_threshold(testing_litellm_router):
     """
     Test that error rate cooldown does NOT trigger on first failure.