diff --git a/docs/my-website/docs/adaptive_router.md b/docs/my-website/docs/adaptive_router.md index 80532f383bb..1e78ad4647a 100644 --- a/docs/my-website/docs/adaptive_router.md +++ b/docs/my-website/docs/adaptive_router.md @@ -131,13 +131,13 @@ Returns current quality estimates per model per request type. Useful for underst "request_type": "analytical_reasoning", "model": "fast", "quality_mean": 0.5, - "samples": 10.0 + "samples": 0 }, { "request_type": "analytical_reasoning", "model": "smart", "quality_mean": 0.95, - "samples": 10.0 + "samples": 0 } ] } @@ -145,7 +145,7 @@ Returns current quality estimates per model per request type. Useful for underst } ``` -`quality_mean` is the key number — it's the router's current estimate of how well that model handles that request type. `samples` counts how many real observations have moved the prior (starts at 10, the cold-start mass). +`quality_mean` is the key number — it's the router's current estimate of how well that model handles that request type. `samples` counts how many real observations have moved the prior (starts at 0; the cold-start prior mass is excluded). ## Known limitations diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 52b5cc7b653..7979b7d09d1 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState { alpha Float beta Float total_samples Int @default(0) - last_updated_at DateTime @default(now()) + last_updated_at DateTime @default(now()) @updatedAt @@id([router_name, request_type, model_name]) } @@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession { last_processed_turn Int @default(-1) clean_credit_awarded Boolean @default(false) terminal_status Int? - last_activity_at DateTime @default(now()) + last_activity_at DateTime @default(now()) @updatedAt @@id([session_id, router_name, model_name]) @@index([last_activity_at]) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 52b5cc7b653..7979b7d09d1 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState { alpha Float beta Float total_samples Int @default(0) - last_updated_at DateTime @default(now()) + last_updated_at DateTime @default(now()) @updatedAt @@id([router_name, request_type, model_name]) } @@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession { last_processed_turn Int @default(-1) clean_credit_awarded Boolean @default(false) terminal_status Int? - last_activity_at DateTime @default(now()) + last_activity_at DateTime @default(now()) @updatedAt @@id([session_id, router_name, model_name]) @@index([last_activity_at]) diff --git a/litellm/router.py b/litellm/router.py index f3ce5985ef5..6c7e73e6801 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -202,6 +202,7 @@ if TYPE_CHECKING: ) from litellm.router_strategy.adaptive_router.adaptive_router import ( AdaptiveRouter, + ) from litellm.router_strategy.quality_router.quality_router import ( QualityRouter, ) @@ -9901,6 +9902,14 @@ class Router: adaptive_router = self.adaptive_routers.get(model) if adaptive_router is not None: return await adaptive_router.async_pre_routing_hook( + model=model, + request_kwargs=request_kwargs, + messages=messages, + input=input, + specific_deployment=specific_deployment, + ) + + ######################################################### # Check if any quality-router should be used ######################################################### if model in self.quality_routers: diff --git a/litellm/router_strategy/adaptive_router/adaptive_router.py b/litellm/router_strategy/adaptive_router/adaptive_router.py index 1e8d02185d7..d6ffd61b7bf 100644 --- a/litellm/router_strategy/adaptive_router/adaptive_router.py +++ b/litellm/router_strategy/adaptive_router/adaptive_router.py @@ -261,7 +261,11 @@ class AdaptiveRouter: "model": model, "alpha": cell.alpha, "beta": cell.beta, - "samples": total, + # Net observations that have moved the posterior, excluding + # the cold-start prior mass. `alpha + beta` would show the + # initial COLD_START_MASS (e.g. 10) before any real traffic + # arrives, which confuses operators reading the endpoint. + "samples": cell.total_samples, "quality_mean": cell.alpha / total if total > 0 else 0.0, } ) diff --git a/schema.prisma b/schema.prisma index 52b5cc7b653..7979b7d09d1 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState { alpha Float beta Float total_samples Int @default(0) - last_updated_at DateTime @default(now()) + last_updated_at DateTime @default(now()) @updatedAt @@id([router_name, request_type, model_name]) } @@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession { last_processed_turn Int @default(-1) clean_credit_awarded Boolean @default(false) terminal_status Int? - last_activity_at DateTime @default(now()) + last_activity_at DateTime @default(now()) @updatedAt @@id([session_id, router_name, model_name]) @@index([last_activity_at]) diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_state_endpoint.py b/tests/test_litellm/router_strategy/adaptive_router/test_state_endpoint.py index 80fa2dc8a57..753a449791b 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_state_endpoint.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_state_endpoint.py @@ -89,7 +89,9 @@ async def test_get_state_snapshot_quality_mean_matches_alpha_over_total(): ) assert cell["alpha"] == expected.alpha assert cell["beta"] == expected.beta - assert cell["samples"] == expected.alpha + expected.beta + # `samples` reports net observations after subtracting the cold-start + # prior mass, so operators aren't misled by the initial value. + assert cell["samples"] == expected.total_samples assert cell["quality_mean"] == pytest.approx(expected_mean)