From 466098c0e3c9713e3dcb61db9cfcec39a7efb908 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Wed, 12 Aug 2026 01:06:58 -0700 Subject: [PATCH] fix(observability): clear the pending-waiter baseline when the query engine restarts The restart guard zeroed the counter deltas but left the waiter baseline in place. A fresh engine's waiter gauge carries no latch, so subtracting the pre-restart baseline hid real waiters and could report zero during the very saturation that caused the restart. --- litellm/proxy/db/db_pool_metrics.py | 7 ++++- .../proxy/db/test_db_pool_metrics.py | 30 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/db/db_pool_metrics.py b/litellm/proxy/db/db_pool_metrics.py index 0b9fd232337..20161614582 100644 --- a/litellm/proxy/db/db_pool_metrics.py +++ b/litellm/proxy/db/db_pool_metrics.py @@ -162,12 +162,17 @@ class DBPoolMetricsSampler: return max(0.0, sample.pending_acquirers - self._pending_baseline) def _to_update(self, sample: DBPoolSample) -> DBPoolMetricsUpdate: - pending: Final = self._corrected_pending_acquirers(sample) previous: Final = self._previous engine_restarted: Final = previous is not None and ( sample.acquire_count_total < previous.acquire_count_total or sample.query_count_total < previous.query_count_total ) + if engine_restarted: + # A fresh engine has an unlatched waiter gauge, so carrying the old + # baseline would subtract a latch that no longer exists and hide + # real waiters. + self._pending_baseline = 0.0 + pending: Final = self._corrected_pending_acquirers(sample) if previous is None or engine_restarted: return DBPoolMetricsUpdate( sample=sample, diff --git a/tests/test_litellm/proxy/db/test_db_pool_metrics.py b/tests/test_litellm/proxy/db/test_db_pool_metrics.py index 6fe8be686af..f4c5c22779f 100644 --- a/tests/test_litellm/proxy/db/test_db_pool_metrics.py +++ b/tests/test_litellm/proxy/db/test_db_pool_metrics.py @@ -263,3 +263,33 @@ async def test_the_raw_engine_reading_is_still_carried_on_the_sample(): assert update is not None assert update.pending_acquirers == 0.0 assert update.sample.pending_acquirers == 4.0 + + +@pytest.mark.asyncio +async def test_an_engine_restart_clears_the_pending_latch(): + """The restart guard zeroes the counter deltas; the waiter baseline has to go + with them. A fresh engine's gauge carries no latch, so keeping the old + baseline would subtract waiters that are really queued and can report zero + during the saturation that caused the restart.""" + clock = _Clock() + client = _Client( + # saturated, 6 waiters of which 4 are latched timeouts + _Metrics(busy=2.0, idle=0.0, wait=6.0, waits=40, queries=40), + # drained: baseline arms at 4 + _Metrics(busy=0.0, idle=2.0, wait=4.0, waits=40, queries=40), + # engine restarted (totals reset) and is saturated again with 3 real waiters + _Metrics(busy=2.0, idle=0.0, wait=3.0, waits=1, queries=1), + ) + sampler = DBPoolMetricsSampler(min_interval_seconds=1.0, monotonic=clock) + + observed = [] + for tick in range(3): + clock.now = tick * 2.0 + update = await sampler.maybe_sample(lambda: client) + assert update is not None + observed.append(update.pending_acquirers) + + assert observed[1] == 0.0, "an idle pool reports no waiters and arms the baseline" + assert observed[2] == 3.0, ( + f"after a restart the fresh gauge must be reported in full, got {observed[2]}" + )