fix(adaptive_router): P2 review items — @updatedAt + snapshot samples

- Mark last_updated_at (AdaptiveRouterState) and last_activity_at
  (AdaptiveRouterSession) with @updatedAt so Prisma refreshes the
  timestamps on every write. Without this the fields stayed frozen at
  INSERT time and the last_activity_at index was misleading for any
  future TTL/eviction logic. Applied to all three schema.prisma copies;
  no migration SQL change needed (Prisma @updatedAt is a client-side
  annotation that doesn't touch DDL).

- get_state_snapshot: report cell.total_samples instead of alpha+beta
  for the 'samples' field. The previous value inflated every cell by
  the COLD_START_MASS prior (e.g. showed 10.0 before any real traffic
  arrived), which confused operators reading /adaptive_router/.../state.
  Updated docs + the snapshot test to match.

Also fixes two pre-existing merge-break syntax errors in router.py
(missing ')' on the AdaptiveRouter TYPE_CHECKING import; truncated
async_pre_routing_hook dispatch call for the adaptive router branch)
that were masking the rest of the file from the interpreter.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Krrish Dholakia 2026-04-21 16:27:01 -07:00
parent c7342bdc4f
commit ecd9a83e61
7 changed files with 26 additions and 11 deletions

View file

@ -131,13 +131,13 @@ Returns current quality estimates per model per request type. Useful for underst
"request_type": "analytical_reasoning",
"model": "fast",
"quality_mean": 0.5,
"samples": 10.0
"samples": 0
},
{
"request_type": "analytical_reasoning",
"model": "smart",
"quality_mean": 0.95,
"samples": 10.0
"samples": 0
}
]
}
@ -145,7 +145,7 @@ Returns current quality estimates per model per request type. Useful for underst
}
```
`quality_mean` is the key number — it's the router's current estimate of how well that model handles that request type. `samples` counts how many real observations have moved the prior (starts at 10, the cold-start mass).
`quality_mean` is the key number — it's the router's current estimate of how well that model handles that request type. `samples` counts how many real observations have moved the prior (starts at 0; the cold-start prior mass is excluded).
## Known limitations

View file

@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState {
alpha Float
beta Float
total_samples Int @default(0)
last_updated_at DateTime @default(now())
last_updated_at DateTime @default(now()) @updatedAt
@@id([router_name, request_type, model_name])
}
@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession {
last_processed_turn Int @default(-1)
clean_credit_awarded Boolean @default(false)
terminal_status Int?
last_activity_at DateTime @default(now())
last_activity_at DateTime @default(now()) @updatedAt
@@id([session_id, router_name, model_name])
@@index([last_activity_at])

View file

@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState {
alpha Float
beta Float
total_samples Int @default(0)
last_updated_at DateTime @default(now())
last_updated_at DateTime @default(now()) @updatedAt
@@id([router_name, request_type, model_name])
}
@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession {
last_processed_turn Int @default(-1)
clean_credit_awarded Boolean @default(false)
terminal_status Int?
last_activity_at DateTime @default(now())
last_activity_at DateTime @default(now()) @updatedAt
@@id([session_id, router_name, model_name])
@@index([last_activity_at])

View file

@ -202,6 +202,7 @@ if TYPE_CHECKING:
)
from litellm.router_strategy.adaptive_router.adaptive_router import (
AdaptiveRouter,
)
from litellm.router_strategy.quality_router.quality_router import (
QualityRouter,
)
@ -9901,6 +9902,14 @@ class Router:
adaptive_router = self.adaptive_routers.get(model)
if adaptive_router is not None:
return await adaptive_router.async_pre_routing_hook(
model=model,
request_kwargs=request_kwargs,
messages=messages,
input=input,
specific_deployment=specific_deployment,
)
#########################################################
# Check if any quality-router should be used
#########################################################
if model in self.quality_routers:

View file

@ -261,7 +261,11 @@ class AdaptiveRouter:
"model": model,
"alpha": cell.alpha,
"beta": cell.beta,
"samples": total,
# Net observations that have moved the posterior, excluding
# the cold-start prior mass. `alpha + beta` would show the
# initial COLD_START_MASS (e.g. 10) before any real traffic
# arrives, which confuses operators reading the endpoint.
"samples": cell.total_samples,
"quality_mean": cell.alpha / total if total > 0 else 0.0,
}
)

View file

@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState {
alpha Float
beta Float
total_samples Int @default(0)
last_updated_at DateTime @default(now())
last_updated_at DateTime @default(now()) @updatedAt
@@id([router_name, request_type, model_name])
}
@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession {
last_processed_turn Int @default(-1)
clean_credit_awarded Boolean @default(false)
terminal_status Int?
last_activity_at DateTime @default(now())
last_activity_at DateTime @default(now()) @updatedAt
@@id([session_id, router_name, model_name])
@@index([last_activity_at])

View file

@ -89,7 +89,9 @@ async def test_get_state_snapshot_quality_mean_matches_alpha_over_total():
)
assert cell["alpha"] == expected.alpha
assert cell["beta"] == expected.beta
assert cell["samples"] == expected.alpha + expected.beta
# `samples` reports net observations after subtracting the cold-start
# prior mass, so operators aren't misled by the initial value.
assert cell["samples"] == expected.total_samples
assert cell["quality_mean"] == pytest.approx(expected_mean)