From ecd9a83e61d0d1007cb0f5c1b81eca49ace5e62c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 21 Apr 2026 16:27:01 -0700 Subject: [PATCH] =?UTF-8?q?fix(adaptive=5Frouter):=20P2=20review=20items?= =?UTF-8?q?=20=E2=80=94=20@updatedAt=20+=20snapshot=20samples?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Mark last_updated_at (AdaptiveRouterState) and last_activity_at (AdaptiveRouterSession) with @updatedAt so Prisma refreshes the timestamps on every write. Without this the fields stayed frozen at INSERT time and the last_activity_at index was misleading for any future TTL/eviction logic. Applied to all three schema.prisma copies; no migration SQL change needed (Prisma @updatedAt is a client-side annotation that doesn't touch DDL). - get_state_snapshot: report cell.total_samples instead of alpha+beta for the 'samples' field. The previous value inflated every cell by the COLD_START_MASS prior (e.g. showed 10.0 before any real traffic arrived), which confused operators reading /adaptive_router/.../state. Updated docs + the snapshot test to match. Also fixes two pre-existing merge-break syntax errors in router.py (missing ')' on the AdaptiveRouter TYPE_CHECKING import; truncated async_pre_routing_hook dispatch call for the adaptive router branch) that were masking the rest of the file from the interpreter. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/my-website/docs/adaptive_router.md | 6 +++--- litellm-proxy-extras/litellm_proxy_extras/schema.prisma | 4 ++-- litellm/proxy/schema.prisma | 4 ++-- litellm/router.py | 9 +++++++++ .../router_strategy/adaptive_router/adaptive_router.py | 6 +++++- schema.prisma | 4 ++-- .../adaptive_router/test_state_endpoint.py | 4 +++- 7 files changed, 26 insertions(+), 11 deletions(-) diff --git a/docs/my-website/docs/adaptive_router.md b/docs/my-website/docs/adaptive_router.md index 80532f383bb..1e78ad4647a 100644 --- a/docs/my-website/docs/adaptive_router.md +++ b/docs/my-website/docs/adaptive_router.md @@ -131,13 +131,13 @@ Returns current quality estimates per model per request type. Useful for underst "request_type": "analytical_reasoning", "model": "fast", "quality_mean": 0.5, - "samples": 10.0 + "samples": 0 }, { "request_type": "analytical_reasoning", "model": "smart", "quality_mean": 0.95, - "samples": 10.0 + "samples": 0 } ] } @@ -145,7 +145,7 @@ Returns current quality estimates per model per request type. Useful for underst } ``` -`quality_mean` is the key number — it's the router's current estimate of how well that model handles that request type. `samples` counts how many real observations have moved the prior (starts at 10, the cold-start mass). +`quality_mean` is the key number — it's the router's current estimate of how well that model handles that request type. `samples` counts how many real observations have moved the prior (starts at 0; the cold-start prior mass is excluded). ## Known limitations diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 52b5cc7b653..7979b7d09d1 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState { alpha Float beta Float total_samples Int @default(0) - last_updated_at DateTime @default(now()) + last_updated_at DateTime @default(now()) @updatedAt @@id([router_name, request_type, model_name]) } @@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession { last_processed_turn Int @default(-1) clean_credit_awarded Boolean @default(false) terminal_status Int? - last_activity_at DateTime @default(now()) + last_activity_at DateTime @default(now()) @updatedAt @@id([session_id, router_name, model_name]) @@index([last_activity_at]) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 52b5cc7b653..7979b7d09d1 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState { alpha Float beta Float total_samples Int @default(0) - last_updated_at DateTime @default(now()) + last_updated_at DateTime @default(now()) @updatedAt @@id([router_name, request_type, model_name]) } @@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession { last_processed_turn Int @default(-1) clean_credit_awarded Boolean @default(false) terminal_status Int? - last_activity_at DateTime @default(now()) + last_activity_at DateTime @default(now()) @updatedAt @@id([session_id, router_name, model_name]) @@index([last_activity_at]) diff --git a/litellm/router.py b/litellm/router.py index f3ce5985ef5..6c7e73e6801 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -202,6 +202,7 @@ if TYPE_CHECKING: ) from litellm.router_strategy.adaptive_router.adaptive_router import ( AdaptiveRouter, + ) from litellm.router_strategy.quality_router.quality_router import ( QualityRouter, ) @@ -9901,6 +9902,14 @@ class Router: adaptive_router = self.adaptive_routers.get(model) if adaptive_router is not None: return await adaptive_router.async_pre_routing_hook( + model=model, + request_kwargs=request_kwargs, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) + + ######################################################### # Check if any quality-router should be used ######################################################### if model in self.quality_routers: diff --git a/litellm/router_strategy/adaptive_router/adaptive_router.py b/litellm/router_strategy/adaptive_router/adaptive_router.py index 1e8d02185d7..d6ffd61b7bf 100644 --- a/litellm/router_strategy/adaptive_router/adaptive_router.py +++ b/litellm/router_strategy/adaptive_router/adaptive_router.py @@ -261,7 +261,11 @@ class AdaptiveRouter: "model": model, "alpha": cell.alpha, "beta": cell.beta, - "samples": total, + # Net observations that have moved the posterior, excluding + # the cold-start prior mass. `alpha + beta` would show the + # initial COLD_START_MASS (e.g. 10) before any real traffic + # arrives, which confuses operators reading the endpoint. + "samples": cell.total_samples, "quality_mean": cell.alpha / total if total > 0 else 0.0, } ) diff --git a/schema.prisma b/schema.prisma index 52b5cc7b653..7979b7d09d1 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState { alpha Float beta Float total_samples Int @default(0) - last_updated_at DateTime @default(now()) + last_updated_at DateTime @default(now()) @updatedAt @@id([router_name, request_type, model_name]) } @@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession { last_processed_turn Int @default(-1) clean_credit_awarded Boolean @default(false) terminal_status Int? - last_activity_at DateTime @default(now()) + last_activity_at DateTime @default(now()) @updatedAt @@id([session_id, router_name, model_name]) @@index([last_activity_at]) diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_state_endpoint.py b/tests/test_litellm/router_strategy/adaptive_router/test_state_endpoint.py index 80fa2dc8a57..753a449791b 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_state_endpoint.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_state_endpoint.py @@ -89,7 +89,9 @@ async def test_get_state_snapshot_quality_mean_matches_alpha_over_total(): ) assert cell["alpha"] == expected.alpha assert cell["beta"] == expected.beta - assert cell["samples"] == expected.alpha + expected.beta + # `samples` reports net observations after subtracting the cold-start + # prior mass, so operators aren't misled by the initial value. + assert cell["samples"] == expected.total_samples assert cell["quality_mean"] == pytest.approx(expected_mean)