From c2cd182c0c433611468631c5c00a65081c91260b Mon Sep 17 00:00:00 2001 From: gvisco Date: Tue, 2 Jun 2026 15:53:04 +0200 Subject: [PATCH] fix(prisma): reset reconnect failure counter on successful watchdog probe _consecutive_reconnect_failures was only reset inside _attempt_reconnect_inside_lock on a successful reconnect cycle. A successful SELECT 1 watchdog probe, which is the steady-state signal that the engine is healthy, never touched the counter. This left it permanently elevated after any transient reconnect failure, causing the next probe failure to escalate to the heavy reconnect path even when the engine had been serving traffic normally for an extended period. --- litellm/proxy/utils.py | 1 + .../proxy/db/test_prisma_self_heal.py | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0e72f47e224..9b6883683df 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4775,6 +4775,7 @@ class PrismaClient: self.db.query_raw("SELECT 1"), timeout=self._db_health_watchdog_probe_timeout_seconds, ) + self._consecutive_reconnect_failures = 0 except asyncio.CancelledError: break except Exception as e: diff --git a/tests/test_litellm/proxy/db/test_prisma_self_heal.py b/tests/test_litellm/proxy/db/test_prisma_self_heal.py index 3f9ba6af3af..8f009d9ec98 100644 --- a/tests/test_litellm/proxy/db/test_prisma_self_heal.py +++ b/tests/test_litellm/proxy/db/test_prisma_self_heal.py @@ -486,3 +486,24 @@ async def test_engine_confirmed_dead_persists_across_failed_heavy_reconnect( # The flag must STILL be True so the next attempt re-enters the heavy # branch instead of silently demoting to the lightweight path. assert client._engine_confirmed_dead is True + + +@pytest.mark.asyncio +async def test_db_health_watchdog_resets_failure_counter_on_successful_probe( + mock_proxy_logging, +): + client = PrismaClient( + database_url="mock://test", proxy_logging_obj=mock_proxy_logging + ) + client.db.query_raw = AsyncMock(return_value=[{"result": 1}]) + client._consecutive_reconnect_failures = 3 + client._db_health_watchdog_interval_seconds = 1 + client._db_health_watchdog_probe_timeout_seconds = 0.2 + + with patch( + "litellm.proxy.utils.asyncio.sleep", + AsyncMock(side_effect=[None, asyncio.CancelledError()]), + ): + await client._db_health_watchdog_loop() + + assert client._consecutive_reconnect_failures == 0