mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
fix(adaptive_router): merge persisted deltas onto cold-start prior on reload
AdaptiveRouter.load_state_from_db overwrote each cold-start Beta prior with the raw persisted row, but AdaptiveRouterUpdateQueue.flush_state_to_db persists accumulated DELTAS (Prisma increment, for multi-pod safety). A satisfaction-only cell persists beta=0 (a negative-only cell persists alpha=0); on the next proxy restart thompson_sample -> random.betavariate(alpha, beta) with a 0 argument raises "ValueError: gammavariate: alpha and beta must be > 0.0", returning HTTP 500 for every request whose request_type has such a cell (pick_best samples every pool model's cell). Merge the persisted deltas onto the cold-start prior instead of overwriting it: keeps the Beta valid (prior.beta >= 0.5) and preserves the tier-seeded prior across restarts. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
35f6961526
commit
431bff3168
1 changed files with 10 additions and 2 deletions
|
|
@ -109,7 +109,7 @@ class AdaptiveRouter:
|
|||
self._cells[(rt, model)] = initial_cell(prefs, rt)
|
||||
|
||||
async def load_state_from_db(self, prisma_client: Any) -> None:
|
||||
"""Override cold-start cells with persisted state. Called once at startup."""
|
||||
"""Merge persisted deltas onto the cold-start prior. Called once at startup."""
|
||||
if prisma_client is None:
|
||||
return
|
||||
try:
|
||||
|
|
@ -125,8 +125,16 @@ class AdaptiveRouter:
|
|||
continue
|
||||
if row.model_name not in self.config.available_models:
|
||||
continue
|
||||
# Persisted alpha/beta are accumulated DELTAS — the flusher writes
|
||||
# increments, not absolute posteriors (for multi-pod safety). Merge
|
||||
# them onto the cold-start prior so the tier bias is preserved AND the
|
||||
# Beta stays valid (prior.beta >= 0.5, so a satisfaction-only delta of
|
||||
# beta=0 can never reload as Beta(alpha, 0) and crash thompson_sample).
|
||||
prefs = self.model_to_prefs.get(row.model_name) or _default_prefs()
|
||||
prior = initial_cell(prefs, rt)
|
||||
self._cells[(rt, row.model_name)] = BanditCell(
|
||||
alpha=row.alpha, beta=row.beta
|
||||
alpha=prior.alpha + row.alpha,
|
||||
beta=prior.beta + row.beta,
|
||||
)
|
||||
loaded += 1
|
||||
verbose_router_logger.info(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue