From 431bff3168ef433ba46ac376c42dac6e2a44b21e Mon Sep 17 00:00:00 2001 From: sha-ir Date: Sun, 31 May 2026 19:54:14 +0000 Subject: [PATCH] fix(adaptive_router): merge persisted deltas onto cold-start prior on reload AdaptiveRouter.load_state_from_db overwrote each cold-start Beta prior with the raw persisted row, but AdaptiveRouterUpdateQueue.flush_state_to_db persists accumulated DELTAS (Prisma increment, for multi-pod safety). A satisfaction-only cell persists beta=0 (a negative-only cell persists alpha=0); on the next proxy restart thompson_sample -> random.betavariate(alpha, beta) with a 0 argument raises "ValueError: gammavariate: alpha and beta must be > 0.0", returning HTTP 500 for every request whose request_type has such a cell (pick_best samples every pool model's cell). Merge the persisted deltas onto the cold-start prior instead of overwriting it: keeps the Beta valid (prior.beta >= 0.5) and preserves the tier-seeded prior across restarts. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../adaptive_router/adaptive_router.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/litellm/router_strategy/adaptive_router/adaptive_router.py b/litellm/router_strategy/adaptive_router/adaptive_router.py index 3bccef36e68..0f8631cb245 100644 --- a/litellm/router_strategy/adaptive_router/adaptive_router.py +++ b/litellm/router_strategy/adaptive_router/adaptive_router.py @@ -109,7 +109,7 @@ class AdaptiveRouter: self._cells[(rt, model)] = initial_cell(prefs, rt) async def load_state_from_db(self, prisma_client: Any) -> None: - """Override cold-start cells with persisted state. Called once at startup.""" + """Merge persisted deltas onto the cold-start prior. Called once at startup.""" if prisma_client is None: return try: @@ -125,8 +125,16 @@ class AdaptiveRouter: continue if row.model_name not in self.config.available_models: continue + # Persisted alpha/beta are accumulated DELTAS — the flusher writes + # increments, not absolute posteriors (for multi-pod safety). Merge + # them onto the cold-start prior so the tier bias is preserved AND the + # Beta stays valid (prior.beta >= 0.5, so a satisfaction-only delta of + # beta=0 can never reload as Beta(alpha, 0) and crash thompson_sample). + prefs = self.model_to_prefs.get(row.model_name) or _default_prefs() + prior = initial_cell(prefs, rt) self._cells[(rt, row.model_name)] = BanditCell( - alpha=row.alpha, beta=row.beta + alpha=prior.alpha + row.alpha, + beta=prior.beta + row.beta, ) loaded += 1 verbose_router_logger.info(