mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
fix(adaptive_router): P2 review items — @updatedAt + snapshot samples
- Mark last_updated_at (AdaptiveRouterState) and last_activity_at (AdaptiveRouterSession) with @updatedAt so Prisma refreshes the timestamps on every write. Without this the fields stayed frozen at INSERT time and the last_activity_at index was misleading for any future TTL/eviction logic. Applied to all three schema.prisma copies; no migration SQL change needed (Prisma @updatedAt is a client-side annotation that doesn't touch DDL). - get_state_snapshot: report cell.total_samples instead of alpha+beta for the 'samples' field. The previous value inflated every cell by the COLD_START_MASS prior (e.g. showed 10.0 before any real traffic arrived), which confused operators reading /adaptive_router/.../state. Updated docs + the snapshot test to match. Also fixes two pre-existing merge-break syntax errors in router.py (missing ')' on the AdaptiveRouter TYPE_CHECKING import; truncated async_pre_routing_hook dispatch call for the adaptive router branch) that were masking the rest of the file from the interpreter. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
c7342bdc4f
commit
ecd9a83e61
7 changed files with 26 additions and 11 deletions
|
|
@ -131,13 +131,13 @@ Returns current quality estimates per model per request type. Useful for underst
|
|||
"request_type": "analytical_reasoning",
|
||||
"model": "fast",
|
||||
"quality_mean": 0.5,
|
||||
"samples": 10.0
|
||||
"samples": 0
|
||||
},
|
||||
{
|
||||
"request_type": "analytical_reasoning",
|
||||
"model": "smart",
|
||||
"quality_mean": 0.95,
|
||||
"samples": 10.0
|
||||
"samples": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -145,7 +145,7 @@ Returns current quality estimates per model per request type. Useful for underst
|
|||
}
|
||||
```
|
||||
|
||||
`quality_mean` is the key number — it's the router's current estimate of how well that model handles that request type. `samples` counts how many real observations have moved the prior (starts at 10, the cold-start mass).
|
||||
`quality_mean` is the key number — it's the router's current estimate of how well that model handles that request type. `samples` counts how many real observations have moved the prior (starts at 0; the cold-start prior mass is excluded).
|
||||
|
||||
## Known limitations
|
||||
|
||||
|
|
|
|||
|
|
@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState {
|
|||
alpha Float
|
||||
beta Float
|
||||
total_samples Int @default(0)
|
||||
last_updated_at DateTime @default(now())
|
||||
last_updated_at DateTime @default(now()) @updatedAt
|
||||
|
||||
@@id([router_name, request_type, model_name])
|
||||
}
|
||||
|
|
@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession {
|
|||
last_processed_turn Int @default(-1)
|
||||
clean_credit_awarded Boolean @default(false)
|
||||
terminal_status Int?
|
||||
last_activity_at DateTime @default(now())
|
||||
last_activity_at DateTime @default(now()) @updatedAt
|
||||
|
||||
@@id([session_id, router_name, model_name])
|
||||
@@index([last_activity_at])
|
||||
|
|
|
|||
|
|
@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState {
|
|||
alpha Float
|
||||
beta Float
|
||||
total_samples Int @default(0)
|
||||
last_updated_at DateTime @default(now())
|
||||
last_updated_at DateTime @default(now()) @updatedAt
|
||||
|
||||
@@id([router_name, request_type, model_name])
|
||||
}
|
||||
|
|
@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession {
|
|||
last_processed_turn Int @default(-1)
|
||||
clean_credit_awarded Boolean @default(false)
|
||||
terminal_status Int?
|
||||
last_activity_at DateTime @default(now())
|
||||
last_activity_at DateTime @default(now()) @updatedAt
|
||||
|
||||
@@id([session_id, router_name, model_name])
|
||||
@@index([last_activity_at])
|
||||
|
|
|
|||
|
|
@ -202,6 +202,7 @@ if TYPE_CHECKING:
|
|||
)
|
||||
from litellm.router_strategy.adaptive_router.adaptive_router import (
|
||||
AdaptiveRouter,
|
||||
)
|
||||
from litellm.router_strategy.quality_router.quality_router import (
|
||||
QualityRouter,
|
||||
)
|
||||
|
|
@ -9901,6 +9902,14 @@ class Router:
|
|||
adaptive_router = self.adaptive_routers.get(model)
|
||||
if adaptive_router is not None:
|
||||
return await adaptive_router.async_pre_routing_hook(
|
||||
model=model,
|
||||
request_kwargs=request_kwargs,
|
||||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
)
|
||||
|
||||
#########################################################
|
||||
# Check if any quality-router should be used
|
||||
#########################################################
|
||||
if model in self.quality_routers:
|
||||
|
|
|
|||
|
|
@ -261,7 +261,11 @@ class AdaptiveRouter:
|
|||
"model": model,
|
||||
"alpha": cell.alpha,
|
||||
"beta": cell.beta,
|
||||
"samples": total,
|
||||
# Net observations that have moved the posterior, excluding
|
||||
# the cold-start prior mass. `alpha + beta` would show the
|
||||
# initial COLD_START_MASS (e.g. 10) before any real traffic
|
||||
# arrives, which confuses operators reading the endpoint.
|
||||
"samples": cell.total_samples,
|
||||
"quality_mean": cell.alpha / total if total > 0 else 0.0,
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1232,7 +1232,7 @@ model LiteLLM_AdaptiveRouterState {
|
|||
alpha Float
|
||||
beta Float
|
||||
total_samples Int @default(0)
|
||||
last_updated_at DateTime @default(now())
|
||||
last_updated_at DateTime @default(now()) @updatedAt
|
||||
|
||||
@@id([router_name, request_type, model_name])
|
||||
}
|
||||
|
|
@ -1261,7 +1261,7 @@ model LiteLLM_AdaptiveRouterSession {
|
|||
last_processed_turn Int @default(-1)
|
||||
clean_credit_awarded Boolean @default(false)
|
||||
terminal_status Int?
|
||||
last_activity_at DateTime @default(now())
|
||||
last_activity_at DateTime @default(now()) @updatedAt
|
||||
|
||||
@@id([session_id, router_name, model_name])
|
||||
@@index([last_activity_at])
|
||||
|
|
|
|||
|
|
@ -89,7 +89,9 @@ async def test_get_state_snapshot_quality_mean_matches_alpha_over_total():
|
|||
)
|
||||
assert cell["alpha"] == expected.alpha
|
||||
assert cell["beta"] == expected.beta
|
||||
assert cell["samples"] == expected.alpha + expected.beta
|
||||
# `samples` reports net observations after subtracting the cold-start
|
||||
# prior mass, so operators aren't misled by the initial value.
|
||||
assert cell["samples"] == expected.total_samples
|
||||
assert cell["quality_mean"] == pytest.approx(expected_mean)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue