scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-05 08:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 136

feat(shadow_eval): compare several auto-routers on one job's sampled traffic (#39028)

Browse source
This commit is contained in:
tin-berri 2026-08-31 21:31:08 -07:00 • committed by GitHub
parent 65a46a5f32
commit bfea8a8c19
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 1124 additions and 420 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm-proxy-extras/litellm_proxy_extras/migrations/20260901000000_shadow_eval_multi_router/migration.sql Normal file
View file

@ -0,0 +1,3 @@
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "router_names" TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[];
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN IF NOT EXISTS "router_name" TEXT;

4
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1531,7 +1531,8 @@ model LiteLLM_ShadowEvalJob {
group_id String // legs of one job share this; the API's job id
target_type String @default("key") // key | team | user
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
router_name String // the auto-router under evaluation, in either direction
router_name String // first (often only) auto-router under evaluation; router_names is the full set
router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name)
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
@ -1555,6 +1556,7 @@ model LiteLLM_ShadowEvalAttempt {
job_id String
request_id String // the judged real request
outcome String // real | shadow | tie | error
router_name String? // the arm this verdict scores; NULL on legacy rows, meaning the job's own router
tier String? // router's tier for the prompt, when classified
real_model String?
shadow_model String?

151
litellm/integrations/shadow_eval_logger.py
View file

@ -1,8 +1,11 @@
"""Shadow Eval Logger: samples a shadowed key's successful LLM requests (chat completions,
Anthropic Messages, and Responses API surfaces, each normalized to chat shape), duplicates
each against the job's other arm in a detached task (the auto-router for a forward job, the
fixed baseline model for a reverse one), blind-judges real vs shadow, and appends one
``LiteLLM_ShadowEvalAttempt`` row (verdict or error) as the feature's only hot-path write.
each through every shadow arm in one detached task (each candidate auto-router for a
forward job, the fixed baseline model for a reverse one), blind-judges real vs each arm,
and appends one ``LiteLLM_ShadowEvalAttempt`` row per arm (verdict or error) as the
feature's only hot-path write. A multi-router job's arms therefore score the identical
sampled requests against the identical real responses, which is what makes their win
rates comparable head-to-head.
Counts, status, and spend derive from those rows at read time, so nothing can disagree
across pods or stop races; the hook reads active jobs through a short-TTL cache."""
@ -498,12 +501,16 @@ def _decision_classifier_cost(metadata: Mapping[str, object]) -> float:
return float(raw) if isinstance(raw, (int, float)) else 0.0
def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: str) -> bool:
"""Whether the router under evaluation served this request, which is what decides
the direction it belongs to. A forward job skips its own router's traffic, since
duplicating it would compare the router to itself: guaranteed ties, judge spend for
zero information. A reverse job samples exactly that traffic and nothing else."""
return _routing_decision(request_metadata).get("router_model_name") == router_name
def _direction_admits(request_metadata: Mapping[str, object], job: "ActiveShadowEvalJob") -> bool:
"""Whether this request belongs to the job's direction. A forward job skips traffic
any of its candidate routers served: duplicating a router's own request compares it
to itself (guaranteed ties), and judging a sibling against another candidate's live
response would score candidates against each other instead of against the incumbent.
A reverse job samples exactly its one router's traffic and nothing else."""
routed_by: Final = _routing_decision(request_metadata).get("router_model_name")
if job.direction == "reverse":
return routed_by == job.router_name
return routed_by not in job.arm_router_names
@dataclass(frozen=True, slots=True)
@ -546,6 +553,7 @@ class ActiveShadowEvalJob(BaseModel):
id: str
router_name: str
router_names: tuple[str, ...] = ()
direction: ShadowEvalDirection = "forward"
baseline_model: str | None = None
shadow_percentage: float
@ -567,12 +575,25 @@ class ActiveShadowEvalJob(BaseModel):
raise ValueError("baseline_model is set for exactly the reverse jobs")
return self
@model_validator(mode="after")
def _reverse_evaluates_one_router(self) -> "ActiveShadowEvalJob":
"""A reverse row naming several routers is unsamplable (there is no one traffic
slice they share) and fails closed."""
if self.direction == "reverse" and len(self.arm_router_names) > 1:
raise ValueError("a reverse job evaluates exactly one router")
return self
@property
def shadow_target(self) -> str:
"""The model the duplicated arm calls: the router itself for a forward job, the
fixed baseline for a reverse one. Total because the validator above pins
def arm_router_names(self) -> tuple[str, ...]:
"""The job's full router set; rows from before router_names existed hold it in
router_name alone. The one place that reading lives on the sampling side."""
return self.router_names or (self.router_name,)
def arm_target(self, arm_router: str) -> str:
"""The model one duplicated arm calls: the candidate router itself for a forward
job, the fixed baseline for a reverse one. Total because the validator above pins
baseline_model to reverse jobs and only those."""
return self.baseline_model or self.router_name
return self.baseline_model or arm_router
def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None:
@ -696,7 +717,7 @@ class ShadowEvalLogger(CustomLogger):
now >= job.ends_at
or job.attempts + self._job_starts.get(job.id, 0) >= job.max_turns
or (job.max_budget is not None and job.spend >= job.max_budget)
or _request_was_routed_by(request_metadata, job.router_name) != (job.direction == "reverse")
or not _direction_admits(request_metadata, job)
):
continue
if not _sample_hits(request_id, job.id, job.shadow_percentage):
@ -773,7 +794,10 @@ class ShadowEvalLogger(CustomLogger):
if self._inflight_shadow_tasks >= _MAX_CONCURRENT_SHADOW_TASKS:
self._record_funnel(job.id, "shed")
continue
self._job_starts[job.id] = self._job_starts.get(job.id, 0) + 1
# One start writes one attempt row per arm, and max_turns is a row
# ceiling, so admission must pre-count every arm or a multi-router
# job overshoots the valve N-fold within a cache generation.
self._job_starts[job.id] = self._job_starts.get(job.id, 0) + len(job.arm_router_names)
self._inflight_shadow_tasks += 1
asyncio.create_task(
self._run_shadow_eval(
@ -812,32 +836,74 @@ class ShadowEvalLogger(CustomLogger):
shadow_params: Mapping[str, object],
parent_metadata: Mapping[str, object],
) -> None:
"""Budget gate -> shadow call -> blind judge -> one attempt row, and every exit
in exactly one coverage bucket: the gates that decline to spend on an admitted
sample (no DB to record into, an over-budget key, an unverifiable or exhausted
eval budget) count it withheld, so eligible traffic still reconciles as
not_sampled + unjudgeable + shed + withheld + attempt rows. The prisma gate sits
above the dispatch so no provider spend happens without a place to record the
outcome, and the budget read lives here rather than in the success hook."""
"""Budget gates once per sampled request, then every router arm in turn: shadow
call -> blind judge -> one attempt row stamped with the arm. The gates that
decline to spend on an admitted sample (no DB to record into, an over-budget key,
an unverifiable or exhausted eval budget) count the REQUEST withheld before any
arm runs, so funnel counters stay per-request and a leg's eligible traffic still
reconciles as not_sampled + unjudgeable + shed + withheld + sampled requests,
where each sampled request writes one attempt row per arm. A budget crossed
mid-loop lets the remaining arms overshoot by one round, the same class of
overshoot as the samples already in flight when the cap is crossed. The prisma
gate sits above the dispatch so no provider spend happens without a place to
record the outcome, and the budget read lives here rather than in the success
hook."""
prisma: Final = self._prisma_provider()
if prisma is None:
self._record_funnel(job.id, "withheld")
return
if await _key_or_team_is_over_budget(parent_metadata):
self._record_funnel(job.id, "withheld")
return
if job.max_budget is not None:
try:
spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget)
except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it
verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e)
self._record_funnel(job.id, "withheld")
return
if spend >= job.max_budget:
self._record_funnel(job.id, "withheld")
return
for arm_router in job.arm_router_names:
await self._run_shadow_arm(
prisma=prisma,
job=job,
arm_router=arm_router,
request_id=request_id,
messages=messages,
real_text=real_text,
real_model=real_model,
real_cost=real_cost,
real_classifier_cost=real_classifier_cost,
real_cache_hit=real_cache_hit,
control_tier=control_tier,
shadow_params=shadow_params,
parent_metadata=parent_metadata,
)
async def _run_shadow_arm(
self,
prisma: "PrismaClient",
job: ActiveShadowEvalJob,
arm_router: str,
request_id: str,
messages: Sequence[Mapping[str, object]],
real_text: str,
real_model: str,
real_cost: float,
real_classifier_cost: float,
real_cache_hit: bool,
control_tier: str | None,
shadow_params: Mapping[str, object],
parent_metadata: Mapping[str, object],
) -> None:
"""One arm's pipeline: shadow call -> blind judge -> one attempt row, every exit
recording this arm's outcome, so one arm's fault never silences a sibling arm."""
try:
if prisma is None:
self._record_funnel(job.id, "withheld")
return
if await _key_or_team_is_over_budget(parent_metadata):
self._record_funnel(job.id, "withheld")
return
if job.max_budget is not None:
try:
spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget)
except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it
verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e)
self._record_funnel(job.id, "withheld")
return
if spend >= job.max_budget:
self._record_funnel(job.id, "withheld")
return
shadow: Final = await self._call_router_shadow(job.shadow_target, messages, shadow_params, parent_metadata)
shadow: Final = await self._call_router_shadow(
job.arm_target(arm_router), messages, shadow_params, parent_metadata
)
except Exception as e: # noqa: BLE001 # detached task: nothing billed yet, record and never raise
verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
await self._record_attempt(
@ -845,6 +911,7 @@ class ShadowEvalLogger(CustomLogger):
job,
request_id,
control_tier,
router_name=arm_router,
outcome="error",
error=f"pipeline error: {e}",
real_cost=real_cost,
@ -858,6 +925,7 @@ class ShadowEvalLogger(CustomLogger):
job,
request_id,
control_tier,
router_name=arm_router,
outcome="error",
error=shadow.error,
shadow_cost=shadow.cost,
@ -882,6 +950,7 @@ class ShadowEvalLogger(CustomLogger):
job,
request_id,
control_tier,
router_name=arm_router,
outcome="error",
error=verdict.error,
shadow=shadow,
@ -898,6 +967,7 @@ class ShadowEvalLogger(CustomLogger):
job,
request_id,
control_tier,
router_name=arm_router,
outcome=verdict.preference,
shadow=shadow,
real_model=real_model,
@ -916,6 +986,7 @@ class ShadowEvalLogger(CustomLogger):
job,
request_id,
control_tier,
router_name=arm_router,
outcome="error",
error=f"pipeline error: {e}",
shadow=shadow,
@ -933,6 +1004,7 @@ class ShadowEvalLogger(CustomLogger):
request_id: str,
control_tier: str | None,
*,
router_name: str,
outcome: str,
real_cost: float,
real_classifier_cost: float,
@ -955,6 +1027,7 @@ class ShadowEvalLogger(CustomLogger):
data={ # mutable-ok: Prisma payload
"job_id": job.id,
"request_id": request_id,
"router_name": router_name,
"outcome": outcome,
"tier": control_tier if job.direction == "reverse" else (shadow.tier if shadow else None),
"real_model": real_model or None,

53
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -833,7 +833,7 @@ def _judge_collisions_for_team(
return tuple(
(role, model)
for role, model in (
*_router_arm_models(llm_router, data.router_name),
*(arm for name in data.router_names for arm in _router_arm_models(llm_router, name)),
*((("baseline", data.baseline_model),) if data.baseline_model is not None else ()),
)
if judge & judge_target(llm_router, model, team_id).models
@ -904,7 +904,7 @@ class _AttemptAggRow(BaseModel):
_ATTEMPT_AGG_ROWS: Final = TypeAdapter(list[_AttemptAggRow])
_ATTEMPT_AGG_SELECT: Final = """
_ATTEMPT_AGG_COLUMNS: Final = """
COUNT(*)::int AS turn_count,
COUNT(*) FILTER (WHERE outcome = 'real')::int AS real_wins,
COUNT(*) FILTER (WHERE outcome = 'shadow')::int AS shadow_wins,
@ -913,15 +913,34 @@ _ATTEMPT_AGG_SELECT: Final = """
COALESCE(SUM(real_cost + real_classifier_cost) FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit), 0)::float AS real_spend,
COALESCE(SUM(shadow_cost + shadow_classifier_cost) FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit), 0)::float AS shadow_spend,
COUNT(*) FILTER (WHERE real_cache_hit)::int AS cache_hit_turns
"""
_ATTEMPT_AGG_SELECT: Final = (
_ATTEMPT_AGG_COLUMNS
+ """
FROM "LiteLLM_ShadowEvalAttempt"
WHERE job_id = ANY($1::text[]) AND outcome != 'error'
GROUP BY 1
"""
)
_ATTEMPT_AGG_BY_TIER_SQL: Final = "SELECT COALESCE(tier, 'UNCLASSIFIED') AS grp," + _ATTEMPT_AGG_SELECT
_ATTEMPT_AGG_BY_MODEL_SQL: Final = "SELECT COALESCE(real_model, 'unknown') AS grp," + _ATTEMPT_AGG_SELECT
_ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT
# Attempt rows from before arm stamping carry no router_name; they belong to the job's
# own router, which the join reads off the leg.
_ATTEMPT_AGG_BY_ROUTER_SQL: Final = (
"SELECT COALESCE(a.router_name, j.router_name) AS grp,"
+ _ATTEMPT_AGG_COLUMNS
+ """
FROM "LiteLLM_ShadowEvalAttempt" a
JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id
WHERE a.job_id = ANY($1::text[]) AND a.outcome != 'error'
GROUP BY 1
"""
)
# These guards derive spend from attempt rows, the cross-pod authority; the sampler also
# reads the live counter, so admission can stop before a row-based guard would fire (safe
# direction, and mid-deploy rows from old pods price as judge-only until the deploy ends).
@ -1060,6 +1079,7 @@ class _LegRow(BaseModel):
target_type: ShadowEvalTargetType
target_id: str
router_name: str
router_names: tuple[str, ...] = ()
direction: ShadowEvalDirection
baseline_model: str | None = None
judge_model: str
@ -1071,6 +1091,12 @@ class _LegRow(BaseModel):
stopped_at: datetime | None = None
stopped_by: str | None = None
@property
def arm_router_names(self) -> tuple[str, ...]:
"""The job's full router set; rows from before router_names existed hold it in
router_name alone. The one place that reading lives on the endpoint side."""
return self.router_names or (self.router_name,)
@field_validator("created_at", "ends_at", "stopped_at")
@classmethod
def _as_aware_utc(cls, value: datetime | None) -> datetime | None:
@ -1123,7 +1149,7 @@ def _group_response(
)
for leg in sorted(legs, key=lambda leg: (leg.target_type, leg.target_id))
),
router_name=first.router_name,
router_names=first.arm_router_names,
direction=first.direction,
baseline_model=first.baseline_model,
judge_model=first.judge_model,
@ -1252,6 +1278,9 @@ async def _shadow_eval_results(
for slice in _slices(by_leg)
}
)
by_router: Final = _ATTEMPT_AGG_ROWS.validate_python(
await _query_raw(prisma_client, _ATTEMPT_AGG_BY_ROUTER_SQL, leg_ids) or ()
)
total_turns: Final = sum(r.turn_count for r in by_tier)
funnel_rows: Final = await _query_raw(prisma_client, _FUNNEL_TOTALS_SQL, leg_ids)
counted: Final = _FunnelTotalsRow.model_validate(funnel_rows[0]) if funnel_rows else None
@ -1261,6 +1290,7 @@ async def _shadow_eval_results(
result: Final = ShadowEvalResult(
by_tier=_slices(by_tier),
by_current_model=_slices(by_model),
by_router=_slices(by_router),
overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns),
overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns),
sampled_real_spend=sum(r.real_spend for r in by_tier),
@ -1314,8 +1344,15 @@ async def start_shadow_eval(
_require_admin_writer(user_api_key_dict, "start a shadow eval")
if prisma_client is None:
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name):
raise HTTPException(status_code=400, detail=f"'{data.router_name}' is not a configured auto-router")
unconfigured: Final = tuple(
name
for name in data.router_names
if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, name)
)
if unconfigured:
raise HTTPException(
status_code=400, detail=f"Not a configured auto-router: {', '.join(repr(n) for n in unconfigured)}"
)
token_rows: Final = (
await _verification_tokens(prisma_client).find_many(
where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter
@ -1416,7 +1453,9 @@ async def start_shadow_eval(
ends_at: Final = now + timedelta(days=data.duration_days)
shared_config: Final = { # mutable-ok: Prisma payload
"group_id": group_id,
"router_name": data.router_name,
# a pre-router_names pod samples router_name alone, so it must be a real arm
"router_name": data.router_names[0],
"router_names": list(data.router_names), # mutable-ok: Prisma payload
"direction": data.direction,
"baseline_model": data.baseline_model,
"judge_model": data.judge_model,
@ -1477,7 +1516,7 @@ async def start_shadow_eval(
)
for target_type, target_id in sorted(requested_targets)
),
router_name=data.router_name,
router_names=data.router_names,
direction=data.direction,
baseline_model=data.baseline_model,
judge_model=data.judge_model,

4
litellm/proxy/schema.prisma
View file

@ -1531,7 +1531,8 @@ model LiteLLM_ShadowEvalJob {
group_id String // legs of one job share this; the API's job id
target_type String @default("key") // key | team | user
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
router_name String // the auto-router under evaluation, in either direction
router_name String // first (often only) auto-router under evaluation; router_names is the full set
router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name)
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
@ -1555,6 +1556,7 @@ model LiteLLM_ShadowEvalAttempt {
job_id String
request_id String // the judged real request
outcome String // real | shadow | tie | error
router_name String? // the arm this verdict scores; NULL on legacy rows, meaning the job's own router
tier String? // router's tier for the prompt, when classified
real_model String?
shadow_model String?

75
litellm/types/management_endpoints/auto_router_endpoints.py
View file

@ -251,8 +251,12 @@ DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5"
# Sample-count ceiling written on every new job: a zero-cost error loop (a shadow arm that
# fails before billing) never consumes spend budget, so it must terminate on count instead.
# A multi-router job writes one attempt row per router arm, so the valve is reached
# proportionally sooner; it is a safety valve, not a sample budget.
SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000
SHADOW_EVAL_MAX_ROUTERS: Final[int] = 4
class StartShadowEvalRequest(BaseModel):
"""Start duplicating one or more targets' traffic for blind comparison against an auto-router.
@ -288,7 +292,24 @@ class StartShadowEvalRequest(BaseModel):
"to across all their teams: JWT requests carrying their subject claim and virtual keys they own"
),
)
router_name: str = Field(description="The auto-router under evaluation, in either direction")
router_name: str | None = Field(
default=None,
description=(
"The auto-router under evaluation, in either direction: the single-router spelling of "
"router_names. Provide exactly one of the two fields"
),
)
router_names: tuple[str, ...] = Field(
default=(),
max_length=SHADOW_EVAL_MAX_ROUTERS,
description=(
"The auto-routers under evaluation, at most "
f"{SHADOW_EVAL_MAX_ROUTERS}. Every sampled request runs through every router listed and each "
"arm is judged independently against the same real response, so routers compare head-to-head "
"on identical traffic. More than one router requires direction 'forward'. After validation "
"this field always carries the full deduplicated set, whichever spelling the caller used"
),
)
direction: ShadowEvalDirection = Field(
default="forward",
description=(
@ -332,7 +353,8 @@ class StartShadowEvalRequest(BaseModel):
"Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with "
"the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval "
"spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight "
"samples can overshoot the cap by one sampling cache window"
"samples can overshoot the cap by one sampling cache window. Every router arm draws from the "
"same per-target budget, so a multi-router job reaches it proportionally sooner"
),
)
@ -373,6 +395,23 @@ class StartShadowEvalRequest(BaseModel):
raise ValueError("baseline_model is only meaningful when direction is 'reverse'")
return self
@model_validator(mode="after")
def _resolve_router_set(self) -> "StartShadowEvalRequest":
"""Whichever spelling the caller used, router_names leaves validation as the full
deduplicated set, so every downstream reader consumes one field."""
if (self.router_name is None) == (not self.router_names):
raise ValueError("provide exactly one of router_name or router_names")
single: Final = () if self.router_name is None else (self.router_name,)
routers: Final = tuple(dict.fromkeys(self.router_names or single))
if not all(name.strip() for name in routers):
raise ValueError("router names must be non-empty strings")
if len(routers) > 1 and self.direction == "reverse":
raise ValueError("a reverse job evaluates one router against baseline_model; pass a single router")
# A returned model_copy is ignored on the __init__ construction path, so the
# normalization must land as a self attribute store to hold for every caller.
self.router_names = routers
return self
class ShadowEvalSlice(BaseModel):
"""Judge outcomes for one slice of a job's verdicts: a router tier, one of the
@ -428,15 +467,28 @@ class ShadowEvalResult(BaseModel):
"and in reverse the models the router itself picked"
)
)
by_router: tuple[ShadowEvalSlice, ...] = Field(
default=(),
description=(
"One slice per router arm, grouped on the router name. Every arm of a multi-router job is "
"judged against the same real responses over the same sampled requests, so these slices "
"compare routers head-to-head: like-for-like win rates and spends on identical traffic. "
"Verdicts from before arm stamping existed count toward the job's own router"
),
)
overall_shadow_win_rate_pct: float
overall_tie_rate_pct: float
sampled_real_spend: float = Field(
default=0.0,
description="USD the real arm billed across all judged turns, cache-served turns excluded",
description=(
"USD the real arm billed across all judged turns, cache-served turns excluded. A judged turn "
"is one (request, router arm) verdict, so a multi-router job counts the real response once per "
"arm it was judged against; per-router comparisons read by_router"
),
)
sampled_shadow_spend: float = Field(
default=0.0,
description="USD the shadow arm billed across the same turns, judge excluded, like for like",
description="USD the shadow arms billed across the same turns, judge excluded, like for like",
)
not_sampled_count: int | None = Field(
default=None,
@ -540,7 +592,13 @@ class ShadowEvalJobResponse(BaseModel):
min_length=1,
description="The targets whose traffic this job evaluates, and only theirs, each with its own budget",
)
router_name: str
router_names: tuple[str, ...] = Field(
min_length=1,
description=(
"Every auto-router this job runs as a shadow arm. Multi-router jobs sample one slice of "
"traffic and judge every arm against the same real responses"
),
)
direction: ShadowEvalDirection = "forward"
baseline_model: str | None = None
judge_model: str
@ -562,6 +620,13 @@ class ShadowEvalJobResponse(BaseModel):
last_error: str | None = Field(default=None, description="Most recent attempt error; detail endpoint only")
results: ShadowEvalResult | None = Field(default=None, description="Stratified verdicts; detail endpoint only")
@computed_field
@property
def router_name(self) -> str:
"""The first router, kept for callers that predate router_names; derived so the
two fields can never disagree."""
return self.router_names[0]
@computed_field
@property
def status(self) -> ShadowEvalStatus:

4
schema.prisma
View file

@ -1531,7 +1531,8 @@ model LiteLLM_ShadowEvalJob {
group_id String // legs of one job share this; the API's job id
target_type String @default("key") // key | team | user
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
router_name String // the auto-router under evaluation, in either direction
router_name String // first (often only) auto-router under evaluation; router_names is the full set
router_names String[] @default([]) // all routers this job runs as shadow arms; empty on legacy rows, whose set is (router_name)
direction String @default("forward") // forward | reverse
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
@ -1555,6 +1556,7 @@ model LiteLLM_ShadowEvalAttempt {
job_id String
request_id String // the judged real request
outcome String // real | shadow | tie | error
router_name String? // the arm this verdict scores; NULL on legacy rows, meaning the job's own router
tier String? // router's tier for the prompt, when classified
real_model String?
shadow_model String?

180
tests/test_litellm/integrations/test_shadow_eval_logger.py
View file

@ -65,6 +65,7 @@ def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash
target_type=target_type,
target_id=target_id,
router_name=job.router_name,
router_names=job.router_names,
direction=job.direction,
baseline_model=job.baseline_model,
shadow_percentage=job.shadow_percentage,
@ -81,6 +82,7 @@ def _router(
shadow_text="shadow answer",
judge_json='{"preference": "A", "confidence": 0.9, "reasoning": "x"}',
classifier_cost=None,
sibling_router_texts=None,
):
"""One mock router serving the shadow call first, the judge call second, told apart by
the internal-origin stamp rather than the model, since a reverse job's shadow arm names
@ -100,6 +102,15 @@ def _router(
decision["classifier_cost"] = classifier_cost
kwargs["metadata"]["routing_decision"] = decision
return {"choices": [{"message": {"content": shadow_text}}], "usage": {"completion_tokens": 5}}
if sibling_router_texts and kwargs["model"] in sibling_router_texts:
kwargs["metadata"]["routing_decision"] = {
"tier_label": "MEDIUM",
"routed_model": f"{kwargs['model']}-pick",
}
return {
"choices": [{"message": {"content": sibling_router_texts[kwargs["model"]]}}],
"usage": {"completion_tokens": 5},
}
return ModelResponse(
model=kwargs["model"],
choices=[{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": shadow_text}}],
@ -1210,29 +1221,36 @@ class TestJobValidation:
{"direction": "reverse"},
{"baseline_model": "baseline-model"},
{"direction": "sideways", "baseline_model": "baseline-model"},
{"direction": "reverse", "baseline_model": "baseline-model", "router_names": ("a", "b")},
],
ids=["reverse-without-baseline", "forward-with-baseline", "unknown-direction"],
ids=["reverse-without-baseline", "forward-with-baseline", "unknown-direction", "reverse-with-router-set"],
)
def test_unsamplable_shapes_are_rejected(self, overrides):
with pytest.raises(ValidationError):
_job(**overrides)
def test_shadow_target_follows_direction(self):
assert _job().shadow_target == "my-router"
assert _reverse_job().shadow_target == "baseline-model"
def test_arm_target_follows_direction(self):
assert _job().arm_target("my-router") == "my-router"
assert _reverse_job().arm_target("my-router") == "baseline-model"
def test_rows_from_before_router_names_carry_their_set_in_router_name(self):
assert _job().arm_router_names == ("my-router",)
assert _job(router_names=("my-router", "alt-router")).arm_router_names == ("my-router", "alt-router")
@pytest.mark.asyncio
class TestDirection:
@pytest.mark.parametrize(
"job,routed_by,sampled",
"job,routed_by,attempt_rows",
[
(_job(), None, True),
(_job(), "my-router", False),
(_job(), "other-router", True),
(_reverse_job(), "my-router", True),
(_reverse_job(), None, False),
(_reverse_job(), "other-router", False),
(_job(), None, 1),
(_job(), "my-router", 0),
(_job(), "other-router", 1),
(_reverse_job(), "my-router", 1),
(_reverse_job(), None, 0),
(_reverse_job(), "other-router", 0),
(_job(router_names=("my-router", "alt-router")), "alt-router", 0),
(_job(router_names=("my-router", "alt-router")), "other-router", 2),
],
ids=[
"forward-samples-unrouted",
@ -1241,20 +1259,24 @@ class TestDirection:
"reverse-samples-its-own-router",
"reverse-skips-unrouted",
"reverse-skips-another-router",
"forward-skips-any-candidates-own-traffic",
"forward-multi-samples-once-per-arm",
],
)
async def test_direction_decides_which_traffic_is_sampled(self, job, routed_by, sampled):
async def test_direction_decides_which_traffic_is_sampled(self, job, routed_by, attempt_rows):
"""The two directions partition the key's traffic: whatever one samples, the other
skips, so a key running both never judges the same turn twice for the same reason."""
skips, so a key running both never judges the same turn twice for the same reason.
A multi-router job extends the forward skip to every candidate: a request one
candidate served must not be judged as the incumbent against another candidate."""
prisma = _prisma()
logger = _logger(router=_router(), prisma=prisma, jobs=(job,))
logger = _logger(router=_router(sibling_router_texts={"alt-router": "alt answer"}), prisma=prisma, jobs=(job,))
await logger.async_log_success_event(
_success_kwargs(request_metadata=_routed_by(routed_by) if routed_by else {}), RESPONSE, None, None
)
await _drain(logger)
assert prisma.db.litellm_shadowevalattempt.create.await_count == int(sampled)
assert prisma.db.litellm_shadowevalattempt.create.await_count == attempt_rows
async def test_reverse_duplicates_against_the_baseline_model(self):
prisma = _prisma()
@ -1316,6 +1338,134 @@ class TestDirection:
assert logger._job_starts == {"forward-job": 1, "reverse-job": 1}
@pytest.mark.asyncio
class TestMultiRouterArms:
async def test_every_arm_judges_the_same_request_and_stamps_its_own_row(self):
"""One sampled request, one row per candidate router, both judged against the same
real response: the paired comparison that makes multi-router win rates comparable."""
prisma = _prisma()
router = _router(sibling_router_texts={"alt-router": "alt answer"})
logger = _logger(router=router, prisma=prisma)
await logger._run_shadow_eval(
job=_job(router_names=("my-router", "alt-router")),
request_id="req-1",
messages=({"role": "user", "content": "hi"},),
real_text="real answer",
real_model="claude-opus",
real_cost=0.001,
real_classifier_cost=0.0,
real_cache_hit=False,
control_tier=None,
shadow_params={},
parent_metadata={},
)
rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list]
assert [row["router_name"] for row in rows] == ["my-router", "alt-router"]
assert {row["request_id"] for row in rows} == {"req-1"}
assert [row["shadow_model"] for row in rows] == ["cheap-model", "alt-router-pick"]
assert all(row["outcome"] in ("real", "shadow", "tie") for row in rows)
assert all(row["real_cost"] == 0.001 for row in rows)
async def test_a_single_router_job_stamps_its_router_on_the_row(self):
prisma = _prisma()
logger = _logger(router=_router(), prisma=prisma)
await logger._run_shadow_eval(
job=_job(),
request_id="req-1",
messages=({"role": "user", "content": "hi"},),
real_text="real answer",
real_model="claude-opus",
real_cost=0.0,
real_classifier_cost=0.0,
real_cache_hit=False,
control_tier=None,
shadow_params={},
parent_metadata={},
)
row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]
assert row["router_name"] == "my-router"
async def test_one_arms_failure_never_silences_the_sibling(self):
prisma = _prisma()
router = _router(sibling_router_texts={"alt-router": "alt answer"})
healthy = router.acompletion.side_effect
async def first_arm_explodes(**kwargs):
if kwargs["model"] == "my-router":
raise RuntimeError("provider exploded")
return await healthy(**kwargs)
router.acompletion.side_effect = first_arm_explodes
logger = _logger(router=router, prisma=prisma)
await logger._run_shadow_eval(
job=_job(router_names=("my-router", "alt-router")),
request_id="req-1",
messages=({"role": "user", "content": "hi"},),
real_text="real answer",
real_model="claude-opus",
real_cost=0.0,
real_classifier_cost=0.0,
real_cache_hit=False,
control_tier=None,
shadow_params={},
parent_metadata={},
)
rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list]
assert [row["router_name"] for row in rows] == ["my-router", "alt-router"]
assert rows[0]["outcome"] == "error"
assert "provider exploded" in rows[0]["error"]
assert rows[1]["outcome"] in ("real", "shadow", "tie")
async def test_the_turn_valve_counts_every_arm_a_start_will_write(self):
"""max_turns is a row ceiling and one sampled request writes one row per arm, so
admission pre-counts the arms: a two-arm job with two turns of budget admits one
request, not two."""
prisma = _prisma()
router = _router(sibling_router_texts={"alt-router": "alt answer"})
logger = _logger(
router=router, prisma=prisma, jobs=(_job(router_names=("my-router", "alt-router"), max_turns=2),)
)
await logger.async_log_success_event(_success_kwargs(request_id="req-1"), RESPONSE, None, None)
await logger.async_log_success_event(_success_kwargs(request_id="req-2"), RESPONSE, None, None)
await _drain(logger)
rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.await_args_list]
assert {row["request_id"] for row in rows} == {"req-1"}
assert len(rows) == 2
async def test_a_withheld_request_runs_no_arm_and_counts_once(self):
"""The budget gates run once per sampled request, before any arm: funnel counters
stay per-request, so coverage math is arm-count independent."""
prisma = _prisma()
router = _router(sibling_router_texts={"alt-router": "alt answer"})
logger = _logger(router=router, prisma=prisma)
await logger._run_shadow_eval(
job=_job(router_names=("my-router", "alt-router"), max_budget=1.0, spend=2.0),
request_id="req-1",
messages=({"role": "user", "content": "hi"},),
real_text="real answer",
real_model="claude-opus",
real_cost=0.0,
real_classifier_cost=0.0,
real_cache_hit=False,
control_tier=None,
shadow_params={},
parent_metadata={},
)
router.acompletion.assert_not_called()
prisma.db.litellm_shadowevalattempt.create.assert_not_called()
assert logger._test_funnel == [("job-1", "withheld")]
@pytest.mark.asyncio
class TestActiveJobsFailClosed:
async def test_a_row_the_sampler_cannot_read_is_dropped_not_guessed(self):

129
tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py
View file

@ -881,6 +881,7 @@ def _leg_record(**overrides: object) -> MagicMock:
"target_type": "key",
"target_id": "key-hash",
"router_name": "my-router",
"router_names": (),
"direction": "forward",
"baseline_model": None,
"judge_model": "anthropic/claude-sonnet-5",
@ -931,6 +932,7 @@ def _shadow_prisma(
legs=(),
agg_rows=None,
by_leg_rows=None,
by_router_rows=None,
known_keys=("key-hash", "key-hash-2"),
key_teams=None,
known_teams=None,
@ -1030,6 +1032,7 @@ def _shadow_prisma(
"target_type",
"target_id",
"router_name",
"router_names",
"direction",
"baseline_model",
"judge_model",
@ -1065,6 +1068,8 @@ def _shadow_prisma(
return [{"judged_count": 10, "error_count": 2, "judge_spend": 0.031}]
if "SELECT job_id AS grp" in sql:
return by_leg_rows if by_leg_rows is not None else []
if "COALESCE(a.router_name" in sql:
return by_router_rows if by_router_rows is not None else []
if 'FROM "LiteLLM_ShadowEvalFunnel"' in sql:
return prisma.funnel_rows
return agg_rows if agg_rows is not None else []
@ -1121,7 +1126,15 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp
prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once()
rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
assert [(row["target_type"], row["target_id"]) for row in rows] == [("key", "key-hash"), ("key", "key-hash-2")]
assert len({frozenset((k, v) for k, v in row.items() if k not in ("target_id", "id")) for row in rows}) == 1
assert (
len(
{
frozenset((k, tuple(v) if isinstance(v, list) else v) for k, v in row.items() if k not in ("target_id", "id"))
for row in rows
}
)
== 1
)
assert len({row["id"] for row in rows}) == len(rows)
assert len({row["group_id"] for row in rows}) == 1
assert all(row["max_turns"] == SHADOW_EVAL_TURN_VALVE and row["created_by"] == "admin" for row in rows)
@ -1138,6 +1151,63 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp
assert all(target.max_turns == SHADOW_EVAL_TURN_VALVE for target in response.targets)
@pytest.mark.asyncio
async def test_start_shadow_eval_multi_router_writes_the_set_on_every_leg(monkeypatch: pytest.MonkeyPatch):
"""A multi-router job stores the full set in router_names and the first router in
router_name, so a rolling-deploy pod that predates router_names still runs a valid
single-arm eval and its unstamped attempt rows attribute to that first router."""
import litellm.proxy.proxy_server as proxy_server
_configure_anthropic_sdk_judge(monkeypatch)
prisma = _shadow_prisma()
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
response = await start_shadow_eval(
_start_request(router_name=None, router_names=("my-router", "classifier-router")), ADMIN
)
rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
assert all(row["router_name"] == "my-router" for row in rows)
assert all(row["router_names"] == ["my-router", "classifier-router"] for row in rows)
assert response.router_names == ("my-router", "classifier-router")
assert response.router_name == "my-router"
@pytest.mark.asyncio
async def test_start_shadow_eval_rejects_an_unconfigured_router_in_the_set(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server
_configure_anthropic_sdk_judge(monkeypatch)
prisma = _shadow_prisma()
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
with pytest.raises(HTTPException, match="not-a-router") as exc:
await start_shadow_eval(_start_request(router_name=None, router_names=("my-router", "not-a-router")), ADMIN)
assert exc.value.status_code == 400
prisma.db.litellm_shadowevaljob.create_many.assert_not_called()
@pytest.mark.asyncio
async def test_judge_collision_is_found_on_every_router_of_the_set(monkeypatch: pytest.MonkeyPatch):
"""The judge-as-candidate guard walks every candidate router: a judge that serves an
arm of the SECOND router still poisons the whole job's win rates."""
import litellm.proxy.proxy_server as proxy_server
_configure_anthropic_sdk_judge(monkeypatch)
prisma = _shadow_prisma()
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
with pytest.raises(HTTPException, match="also an arm") as exc:
await start_shadow_eval(_start_request(router_name=None, router_names=("my-router", "sonnet-router")), ADMIN)
assert exc.value.status_code == 400
prisma.db.litellm_shadowevaljob.create_many.assert_not_called()
@pytest.mark.asyncio
async def test_start_shadow_eval_rejects_an_uncredentialed_sdk_judge(monkeypatch: pytest.MonkeyPatch) -> None:
import litellm
@ -1798,6 +1868,63 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke
assert error_where == {"job_id": {"in": ["leg-1", "leg-2"]}, "outcome": "error"}
@pytest.mark.asyncio
async def test_get_shadow_eval_job_slices_results_per_router(monkeypatch: pytest.MonkeyPatch):
"""A multi-router job's detail carries one slice per arm, aggregated by the arm
stamped on each attempt row, with unstamped legacy rows attributed to the job's own
router by the read (the COALESCE against the leg's router_name)."""
import litellm.proxy.proxy_server as proxy_server
def agg(grp: str, wins: int) -> dict[str, object]:
return {
"grp": grp,
"turn_count": 4,
"real_wins": 4 - wins,
"shadow_wins": wins,
"ties": 0,
"avg_confidence": 0.8,
"real_spend": 0.08,
"shadow_spend": 0.02,
"cache_hit_turns": 0,
}
prisma = _shadow_prisma(
legs=[_leg_record(router_names=("my-router", "alt-router"))],
agg_rows=[agg("SIMPLE", 3)],
by_router_rows=[agg("my-router", 1), agg("alt-router", 3)],
)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
response = await get_shadow_eval_job("job-1", VIEWER)
assert response.router_names == ("my-router", "alt-router")
assert response.router_name == "my-router"
assert [(s.group, s.shadow_win_rate_pct) for s in response.results.by_router] == [
("my-router", 25.0),
("alt-router", 75.0),
]
router_sql = next(
call.args[0] for call in prisma.db.query_raw.await_args_list if "COALESCE(a.router_name" in call.args[0]
)
assert "COALESCE(a.router_name, j.router_name)" in router_sql
assert 'JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id' in router_sql
assert "a.job_id = ANY($1::text[])" in router_sql
@pytest.mark.asyncio
async def test_job_responses_resolve_router_names_with_legacy_fallback(monkeypatch: pytest.MonkeyPatch):
"""Rows from before router_names existed carry their whole set in router_name."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(legs=[_leg_record(router_names=())])
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
response = await get_shadow_eval_job("job-1", VIEWER)
assert response.router_names == ("my-router",)
assert response.router_name == "my-router"
@pytest.mark.asyncio
async def test_get_shadow_eval_job_404s_and_gates_on_role(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server

126
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx
View file

@ -102,6 +102,7 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
job_id: "job-1",
status: "running",
router_name: "claude-auto",
router_names: ["claude-auto"],
direction: "forward",
baseline_model: null,
judge_model: "anthropic/claude-sonnet-5",
@ -436,7 +437,7 @@ describe("ShadowEvalSection", () => {
await user.click(within(keyList).getByText("prod-alpha"));
await user.click(keyInput);
await user.click(within(keyList).getByText("staging-beta"));
await user.click(screen.getByPlaceholderText("Select an auto-router"));
await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers"));
await user.click(await screen.findByText("gpt-auto"));
expect(screen.getByText("Start shadow eval")).toBeDisabled();
@ -449,7 +450,7 @@ describe("ShadowEvalSection", () => {
api_key_ids: ["hash-alpha", "hash-beta"],
team_ids: [],
user_ids: [],
router_name: "gpt-auto",
router_names: ["gpt-auto"],
direction: "forward",
shadow_percentage: 10,
duration_days: 7,
@ -469,7 +470,7 @@ describe("ShadowEvalSection", () => {
await user.click(screen.getByPlaceholderText("Search teams by alias"));
const teamList = await screen.findByTestId("paginated-multi-select-list");
await user.click(within(teamList).getByText("engineering"));
await user.click(screen.getByPlaceholderText("Select an auto-router"));
await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers"));
await user.click(await screen.findByText("gpt-auto"));
await user.click(screen.getByPlaceholderText("Select a judge model"));
await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ }));
@ -479,7 +480,7 @@ describe("ShadowEvalSection", () => {
api_key_ids: [],
team_ids: ["team-eng"],
user_ids: [],
router_name: "gpt-auto",
router_names: ["gpt-auto"],
direction: "forward",
shadow_percentage: 10,
duration_days: 7,
@ -501,7 +502,7 @@ describe("ShadowEvalSection", () => {
await user.click(screen.getByPlaceholderText("Search keys by alias"));
const keyList = await screen.findByTestId("paginated-multi-select-list");
await user.click(within(keyList).getByText("prod-alpha"));
await user.click(screen.getByPlaceholderText("Select an auto-router"));
await user.click(screen.getByPlaceholderText("Select up to 4 auto-routers"));
await user.click(await screen.findByText("gpt-auto"));
await user.click(screen.getByPlaceholderText("Select a judge model"));
await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ }));
@ -517,7 +518,7 @@ describe("ShadowEvalSection", () => {
api_key_ids: ["hash-alpha"],
team_ids: [],
user_ids: [],
router_name: "gpt-auto",
router_names: ["gpt-auto"],
direction: "reverse",
baseline_model: "prod-claude",
shadow_percentage: 10,
@ -528,6 +529,119 @@ describe("ShadowEvalSection", () => {
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
});
it("submits every picked auto-router so one job compares them on the same traffic", async () => {
const user = userEvent.setup();
const { start } = mockHooks({});
render(<ShadowEvalSection />);
await user.click(screen.getByPlaceholderText("Search keys by alias"));
const keyList = await screen.findByTestId("paginated-multi-select-list");
await user.click(within(keyList).getByText("prod-alpha"));
const routerInput = screen.getByPlaceholderText("Select up to 4 auto-routers");
await user.click(routerInput);
await user.click(await screen.findByText("gpt-auto"));
await user.click(routerInput);
await user.click(await screen.findByText("claude-auto"));
expect(
screen.getByText("Every router sees the same sampled requests, judged against the same live responses"),
).toBeInTheDocument();
await user.click(screen.getByPlaceholderText("Select a judge model"));
await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ }));
await user.click(screen.getByText("Start shadow eval"));
const expectedBody = {
api_key_ids: ["hash-alpha"],
team_ids: [],
user_ids: [],
router_names: ["gpt-auto", "claude-auto"],
direction: "forward",
shadow_percentage: 10,
duration_days: 7,
max_budget: 10,
judge_model: "anthropic/claude-sonnet-5",
};
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
});
it("blocks starting a reverse job with more than one router and says why", async () => {
const user = userEvent.setup();
mockHooks({});
render(<ShadowEvalSection />);
await user.click(screen.getByPlaceholderText("Search keys by alias"));
const keyList = await screen.findByTestId("paginated-multi-select-list");
await user.click(within(keyList).getByText("prod-alpha"));
const routerInput = screen.getByPlaceholderText("Select up to 4 auto-routers");
await user.click(routerInput);
await user.click(await screen.findByText("gpt-auto"));
await user.click(routerInput);
await user.click(await screen.findByText("claude-auto"));
await user.click(screen.getByText("Adoption check: key's traffic vs the router"));
await user.click(await screen.findByText("Regression check: router's picks vs a baseline"));
await user.click(screen.getByPlaceholderText("Select a judge model"));
await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ }));
await user.click(screen.getByPlaceholderText("Select a baseline model"));
await user.click(screen.getByRole("option", { name: /prod-claude/ }));
expect(screen.getByText("A regression check compares one router to its baseline")).toBeInTheDocument();
expect(screen.getByText("Start shadow eval")).toBeDisabled();
});
it("renders a per-router comparison table only when the job ran several routers", () => {
const routerSlice = (group: string, wins: number) => ({
group,
turn_count: 20,
real_win_rate_pct: 100 - wins - 10,
shadow_win_rate_pct: wins,
tie_rate_pct: 10,
avg_judge_confidence: 0.8,
real_spend: 0.4,
shadow_spend: 0.2,
cache_hit_turns: 0,
});
const base = job();
const multi = job({
router_names: ["claude-auto", "gpt-auto"],
results: { ...base.results!, by_router: [routerSlice("claude-auto", 40), routerSlice("gpt-auto", 70)] },
});
mockHooks({ jobs: [multi], detailsById: { "job-1": multi } });
render(<ShadowEvalSection />);
expect(screen.getByText("Router")).toBeInTheDocument();
const rows = screen.getAllByRole("row").map((row) => row.textContent ?? "");
expect(rows.some((text) => text.includes("claude-auto") && text.includes("40.0%"))).toBe(true);
expect(rows.some((text) => text.includes("gpt-auto") && text.includes("70.0%"))).toBe(true);
expect(
screen.getByText(
(_, element) =>
element?.textContent === "Shadowing 10% of prod-alpha traffic via claude-auto, gpt-auto" &&
element.tagName === "P",
),
).toBeInTheDocument();
});
it("renders a job from an older proxy that predates router_names", () => {
const legacy = { ...job(), router_names: undefined } as unknown as ShadowEvalJob;
mockHooks({ jobs: [legacy], detailsById: { "job-1": legacy } });
render(<ShadowEvalSection />);
expect(
screen.getByText(
(_, element) =>
element?.textContent === "Shadowing 10% of prod-alpha traffic via claude-auto" && element.tagName === "P",
),
).toBeInTheDocument();
});
it("keeps the per-router table hidden for a single-router job", () => {
const base = job();
const single = job({ results: { ...base.results!, by_router: [] } });
mockHooks({ jobs: [single], detailsById: { "job-1": single } });
render(<ShadowEvalSection />);
expect(screen.queryByText("Router")).not.toBeInTheDocument();
});
it("flips the arm labels and headline for a reverse job's results", () => {
const j = job({ direction: "reverse", baseline_model: "openai/gpt-4o" });
mockHooks({ jobs: [j], detailsById: { "job-1": j } });

348
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx
View file

@ -2,32 +2,21 @@
import React, { useMemo, useState } from "react";
import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels";
import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect";
import TeamMultiSelect from "@/components/common_components/team_multi_select";
import { userOptionLabel } from "@/components/common_components/UserDropdown";
import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { CircleHelp } from "lucide-react";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Card } from "@/components/ui/card";
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
import { ApiError } from "@/lib/http/client";
import { usd } from "./costOptimizationUtils";
import { StartForm } from "./ShadowEvalStartForm";
import {
useShadowEvalJob,
useShadowEvalJobs,
useStartShadowEval,
useStopShadowEval,
type ShadowEvalJob,
type ShadowEvalJobTarget,
@ -96,17 +85,19 @@ const targetStatus = (job: ShadowEvalJob, target: ShadowEvalJobTarget): string =
return target.stopped_at != null ? "stopped" : "running";
};
const jobRouters = (job: ShadowEvalJob): string => (job.router_names ?? [job.router_name]).join(", ");
const jobHeadline = (job: ShadowEvalJob): React.ReactNode =>
job.direction === "reverse" ? (
<>
Comparing <span className="font-mono text-xs">{job.router_name}</span> to{" "}
Comparing <span className="font-mono text-xs">{jobRouters(job)}</span> to{" "}
<span className="font-mono text-xs">{job.baseline_model}</span> on {job.shadow_percentage}% of{" "}
<span className="font-mono text-xs">{shadowedTargetsLabel(job)}</span> traffic
</>
) : (
<>
Shadowing {job.shadow_percentage}% of <span className="font-mono text-xs">{shadowedTargetsLabel(job)}</span>{" "}
traffic via <span className="font-mono text-xs">{job.router_name}</span>
traffic via <span className="font-mono text-xs">{jobRouters(job)}</span>
</>
);
@ -352,6 +343,11 @@ const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({
<CostComparison direction={job.direction} results={results} />
</div>
<VerdictBar direction={job.direction} results={results} />
{(results.by_router ?? []).length > 1 && (
<div className="border-b">
<SliceTable groupHeader="Router" direction={job.direction} slices={results.by_router ?? []} />
</div>
)}
{results.by_current_model.length > 0 && (
<SliceTable
groupHeader={job.direction === "reverse" ? "Router pick" : "Compared against"}
@ -410,328 +406,6 @@ const JobResults: React.FC<{
);
};
const RECOMMENDED_JUDGE_MODELS = ["anthropic/claude-sonnet-5", "openai/gpt-4o", "gemini/gemini-2.5-pro"] as const;
interface CostMapEntry {
litellm_provider?: string;
mode?: string;
}
const useChatModelNames = (): string[] => {
const { data: costMap } = useModelCostMap();
return useMemo(() => {
if (!costMap) return [];
const chatModels = Object.entries(costMap as Record<string, CostMapEntry>)
.filter(([, value]) => value?.mode === "chat" && value?.litellm_provider)
.map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`));
return [...new Set(chatModels)].toSorted((a, b) => a.localeCompare(b));
}, [costMap]);
};
const useJudgeModelOptions = (): SearchSelectOption[] => {
const chatModels = useChatModelNames();
return useMemo(() => {
const pinned: SearchSelectOption[] = RECOMMENDED_JUDGE_MODELS.map((model) => ({
label: model,
value: model,
sublabel: "Recommended",
}));
const pinnedNames = new Set<string>(RECOMMENDED_JUDGE_MODELS);
const rest = chatModels.filter((model) => !pinnedNames.has(model)).map((model) => ({ label: model, value: model }));
return [...pinned, ...rest];
}, [chatModels]);
};
const useBaselineModelOptions = (): SearchSelectOption[] => {
const configuredGroups = usePlainModelGroups();
const chatModels = useChatModelNames();
return useMemo(() => {
const configured = [...configuredGroups]
.toSorted((a, b) => a.localeCompare(b))
.map((model) => ({ label: model, value: model, sublabel: "Configured on this gateway" }));
const rest = chatModels
.filter((model) => !configuredGroups.has(model))
.map((model) => ({ label: model, value: model }));
return [...configured, ...rest];
}, [configuredGroups, chatModels]);
};
const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[] = [
{ value: "forward", label: "Adoption check: key's traffic vs the router" },
{ value: "reverse", label: "Regression check: router's picks vs a baseline" },
] as const;
const START_FORM_DESCRIPTION: Record<ShadowEvalDirection, string> = {
forward:
"Duplicates a sampled slice of the selected targets' traffic (keys, teams, or users) through the auto-router and has an LLM judge compare both answers blind. Each target gets its own spend budget. The router's answers are never served to users; judge calls bill to the sampled traffic's own identity.",
reverse:
"Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each target gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the sampled traffic's own identity.",
};
const DURATION_OPTIONS = [
{ value: "1", label: "1 day" },
{ value: "3", label: "3 days" },
{ value: "7", label: "7 days" },
{ value: "14", label: "14 days" },
{ value: "30", label: "30 days" },
] as const;
const Field: React.FC<{ label: string; htmlFor?: string; className?: string; children: React.ReactNode }> = ({
label,
htmlFor,
className,
children,
}) => (
<div className={`space-y-1.5 ${className ?? ""}`}>
<Label htmlFor={htmlFor} className="text-xs">
{label}
</Label>
{children}
</div>
);
const KeySelect: React.FC<{ value: string[]; onChange: (tokens: string[]) => void }> = ({ value, onChange }) => {
const [search, setSearch] = useState("");
const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteKeys(50, {
selectedKeyAlias: search || null,
});
const options = useMemo<SearchSelectOption[]>(
() =>
(data?.pages ?? [])
.flatMap((page) => page.keys)
.map((key) => ({
label: key.key_alias || key.key_name || key.token,
value: key.token,
sublabel: key.token,
})),
[data],
);
return (
<PaginatedMultiSelect
inputId="shadow-eval-key"
options={options}
value={value}
onValueChange={onChange}
onSearchChange={setSearch}
onLoadMore={() => void fetchNextPage()}
hasNextPage={hasNextPage}
isFetchingNextPage={isFetchingNextPage}
isLoading={isPending}
placeholder="Search keys by alias"
emptyText="No matching keys"
errorText={isError ? "Keys could not be loaded. Refresh the page to retry." : undefined}
/>
);
};
const UserSelect: React.FC<{ value: string[]; onChange: (ids: string[]) => void }> = ({ value, onChange }) => {
const [search, setSearch] = useState("");
const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteUsers(
50,
search || undefined,
);
const options = useMemo<SearchSelectOption[]>(
() =>
Array.from(
new Map(
(data?.pages ?? [])
.flatMap((page) => page.users)
.map((user) => [user.user_id, { label: userOptionLabel(user), value: user.user_id }] as const),
).values(),
),
[data],
);
return (
<PaginatedMultiSelect
inputId="shadow-eval-user"
options={options}
value={value}
onValueChange={onChange}
onSearchChange={setSearch}
onLoadMore={() => void fetchNextPage()}
hasNextPage={hasNextPage}
isFetchingNextPage={isFetchingNextPage}
isLoading={isPending}
placeholder="Search users by email"
emptyText="No matching users"
errorText={isError ? "Users could not be loaded. Refresh the page to retry." : undefined}
/>
);
};
const StartForm: React.FC = () => {
const { accessToken } = useAuthorized();
const [apiKeyIds, setApiKeyIds] = useState<string[]>([]);
const [teamIds, setTeamIds] = useState<string[]>([]);
const [userIds, setUserIds] = useState<string[]>([]);
const [routerName, setRouterName] = useState("");
const [direction, setDirection] = useState<ShadowEvalDirection>("forward");
const [baselineModel, setBaselineModel] = useState("");
const [percentage, setPercentage] = useState("10");
const [durationDays, setDurationDays] = useState("7");
const [judgeModel, setJudgeModel] = useState("");
const [maxBudget, setMaxBudget] = useState("10");
const { data: autoRouters } = useAutoRouters();
const judgeModelOptions = useJudgeModelOptions();
const baselineModelOptions = useBaselineModelOptions();
const start = useStartShadowEval();
const routerOptions = useMemo<SearchSelectOption[]>(() => {
const names = new Set(
(autoRouters ?? []).map((deployment) => deployment.model_name).filter((name): name is string => Boolean(name)),
);
return [...names].toSorted().map((name) => ({ label: name, value: name }));
}, [autoRouters]);
const parsedPct = Number.parseFloat(percentage);
const percentageValid = parsedPct >= 0.1 && parsedPct <= 100;
const parsedMaxBudget = Number.parseFloat(maxBudget);
const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000;
const baselinePicked = direction === "forward" || baselineModel !== "";
const targetsPicked = apiKeyIds.length + teamIds.length + userIds.length > 0;
const filled = targetsPicked && [routerName, judgeModel].every((field) => field !== "") && baselinePicked;
const boundsValid = percentageValid && maxBudgetValid;
const valid = Boolean(accessToken) && filled && boundsValid;
const handleStart = () => {
const startBody = {
api_key_ids: apiKeyIds,
team_ids: teamIds,
user_ids: userIds,
router_name: routerName,
direction,
...(direction === "reverse" ? { baseline_model: baselineModel } : {}),
shadow_percentage: parsedPct,
duration_days: Number.parseInt(durationDays, 10),
max_budget: parsedMaxBudget,
judge_model: judgeModel,
};
start.mutate(startBody);
};
return (
<Card size="sm">
<CardHeader>
<CardTitle className="text-sm font-medium text-foreground">Start a shadow eval</CardTitle>
<p className="text-xs text-muted-foreground">{START_FORM_DESCRIPTION[direction]}</p>
</CardHeader>
<CardContent className="space-y-3">
<div className="grid gap-3 sm:grid-cols-3">
<Field label="Direction">
<Select
value={direction}
onValueChange={(v: string | null) => setDirection(v === "reverse" ? "reverse" : "forward")}
>
<SelectTrigger className="w-full">
<SelectValue>{DIRECTION_OPTIONS.find((o) => o.value === direction)?.label}</SelectValue>
</SelectTrigger>
<SelectContent>
{DIRECTION_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
</Field>
<Field label="Keys to shadow" htmlFor="shadow-eval-key">
<KeySelect value={apiKeyIds} onChange={setApiKeyIds} />
</Field>
<Field label="Teams to shadow">
<TeamMultiSelect value={teamIds} onChange={setTeamIds} placeholder="Search teams by alias" />
</Field>
<Field label="Users to shadow" htmlFor="shadow-eval-user">
<UserSelect value={userIds} onChange={setUserIds} />
</Field>
<Field label="Auto-router">
<SearchSelect
options={routerOptions}
value={routerName}
onValueChange={setRouterName}
placeholder="Select an auto-router"
emptyText="No auto-routers configured"
/>
</Field>
<Field label="Traffic sampled" htmlFor="shadow-eval-pct">
<div className="flex items-center gap-2">
<Input
id="shadow-eval-pct"
type="number"
min={0.1}
max={100}
step={0.1}
className="w-24"
value={percentage}
onChange={(e) => setPercentage(e.target.value)}
/>
<span className="text-sm text-muted-foreground">% of traffic</span>
</div>
<div>
{percentage.trim() !== "" && !percentageValid && (
<p className="text-xs text-destructive">Enter a value from 0.1 to 100</p>
)}
</div>
</Field>
<Field label="Duration">
<Select value={durationDays} onValueChange={(v: string | null) => setDurationDays(v ?? "7")}>
<SelectTrigger className="w-full">
<SelectValue>{DURATION_OPTIONS.find((o) => o.value === durationDays)?.label}</SelectValue>
</SelectTrigger>
<SelectContent>
{DURATION_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
</Field>
<Field label="Spend budget">
<div className="flex items-center gap-2">
<span className="text-sm text-muted-foreground">$</span>
<Input
type="number"
min={0.01}
max={10000}
step={0.01}
className="w-24"
value={maxBudget}
onChange={(e) => setMaxBudget(e.target.value)}
/>
<span className="text-sm text-muted-foreground">max shadow + judge spend, per target</span>
</div>
{maxBudget.trim() !== "" && !maxBudgetValid && (
<p className="text-xs text-destructive">Enter a value from 0.01 to 10000</p>
)}
</Field>
{direction === "reverse" && (
<Field label="Baseline model">
<SearchSelect
options={baselineModelOptions}
value={baselineModel}
onValueChange={setBaselineModel}
placeholder="Select a baseline model"
emptyText="No chat models available"
/>
</Field>
)}
<Field label="Judge model" className="sm:col-span-2">
<SearchSelect
options={judgeModelOptions}
value={judgeModel}
onValueChange={setJudgeModel}
placeholder="Select a judge model"
emptyText="No chat models available"
/>
</Field>
</div>
<Button disabled={!valid || start.isPending} onClick={handleStart}>
{start.isPending ? "Starting..." : "Start shadow eval"}
</Button>
</CardContent>
</Card>
);
};
const previousSummary = (job: ShadowEvalJob): string => {
const results = job.results;
if (results) return pct(routerMatchedOrBeatPct(job.direction, results));

432
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalStartForm.tsx Normal file
View file

@ -0,0 +1,432 @@
"use client";
import React, { useMemo, useState } from "react";
import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers";
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels";
import { MultiSelect } from "@/components/shared/MultiSelect";
import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect";
import TeamMultiSelect from "@/components/common_components/team_multi_select";
import { userOptionLabel } from "@/components/common_components/UserDropdown";
import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect";
import { Button } from "@/components/ui/button";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { useStartShadowEval, type ShadowEvalJob } from "./useShadowEval";
type ShadowEvalDirection = ShadowEvalJob["direction"];
const MAX_ROUTERS = 4;
const RECOMMENDED_JUDGE_MODELS = ["anthropic/claude-sonnet-5", "openai/gpt-4o", "gemini/gemini-2.5-pro"] as const;
interface CostMapEntry {
litellm_provider?: string;
mode?: string;
}
const useChatModelNames = (): string[] => {
const { data: costMap } = useModelCostMap();
return useMemo(() => {
if (!costMap) return [];
const chatModels = Object.entries(costMap as Record<string, CostMapEntry>)
.filter(([, value]) => value?.mode === "chat" && value?.litellm_provider)
.map(([key, value]) => (key.startsWith(`${value.litellm_provider}/`) ? key : `${value.litellm_provider}/${key}`));
return [...new Set(chatModels)].toSorted((a, b) => a.localeCompare(b));
}, [costMap]);
};
const useJudgeModelOptions = (): SearchSelectOption[] => {
const chatModels = useChatModelNames();
return useMemo(() => {
const pinned: SearchSelectOption[] = RECOMMENDED_JUDGE_MODELS.map((model) => ({
label: model,
value: model,
sublabel: "Recommended",
}));
const pinnedNames = new Set<string>(RECOMMENDED_JUDGE_MODELS);
const rest = chatModels.filter((model) => !pinnedNames.has(model)).map((model) => ({ label: model, value: model }));
return [...pinned, ...rest];
}, [chatModels]);
};
const useBaselineModelOptions = (): SearchSelectOption[] => {
const configuredGroups = usePlainModelGroups();
const chatModels = useChatModelNames();
return useMemo(() => {
const configured = [...configuredGroups]
.toSorted((a, b) => a.localeCompare(b))
.map((model) => ({ label: model, value: model, sublabel: "Configured on this gateway" }));
const rest = chatModels
.filter((model) => !configuredGroups.has(model))
.map((model) => ({ label: model, value: model }));
return [...configured, ...rest];
}, [configuredGroups, chatModels]);
};
const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[] = [
{ value: "forward", label: "Adoption check: key's traffic vs the router" },
{ value: "reverse", label: "Regression check: router's picks vs a baseline" },
] as const;
const START_FORM_DESCRIPTION: Record<ShadowEvalDirection, string> = {
forward:
"Duplicates a sampled slice of the selected targets' traffic (keys, teams, or users) through the auto-router and has an LLM judge compare both answers blind. Each target gets its own spend budget. The router's answers are never served to users; judge calls bill to the sampled traffic's own identity.",
reverse:
"Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each target gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the sampled traffic's own identity.",
};
const DURATION_OPTIONS = [
{ value: "1", label: "1 day" },
{ value: "3", label: "3 days" },
{ value: "7", label: "7 days" },
{ value: "14", label: "14 days" },
{ value: "30", label: "30 days" },
] as const;
const Field: React.FC<{ label: string; htmlFor?: string; className?: string; children: React.ReactNode }> = ({
label,
htmlFor,
className,
children,
}) => (
<div className={`space-y-1.5 ${className ?? ""}`}>
<Label htmlFor={htmlFor} className="text-xs">
{label}
</Label>
{children}
</div>
);
const KeySelect: React.FC<{ value: string[]; onChange: (tokens: string[]) => void }> = ({ value, onChange }) => {
const [search, setSearch] = useState("");
const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteKeys(50, {
selectedKeyAlias: search || null,
});
const options = useMemo<SearchSelectOption[]>(
() =>
(data?.pages ?? [])
.flatMap((page) => page.keys)
.map((key) => ({
label: key.key_alias || key.key_name || key.token,
value: key.token,
sublabel: key.token,
})),
[data],
);
return (
<PaginatedMultiSelect
inputId="shadow-eval-key"
options={options}
value={value}
onValueChange={onChange}
onSearchChange={setSearch}
onLoadMore={() => void fetchNextPage()}
hasNextPage={hasNextPage}
isFetchingNextPage={isFetchingNextPage}
isLoading={isPending}
placeholder="Search keys by alias"
emptyText="No matching keys"
errorText={isError ? "Keys could not be loaded. Refresh the page to retry." : undefined}
/>
);
};
const UserSelect: React.FC<{ value: string[]; onChange: (ids: string[]) => void }> = ({ value, onChange }) => {
const [search, setSearch] = useState("");
const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteUsers(
50,
search || undefined,
);
const options = useMemo<SearchSelectOption[]>(
() =>
Array.from(
new Map(
(data?.pages ?? [])
.flatMap((page) => page.users)
.map((user) => [user.user_id, { label: userOptionLabel(user), value: user.user_id }] as const),
).values(),
),
[data],
);
return (
<PaginatedMultiSelect
inputId="shadow-eval-user"
options={options}
value={value}
onValueChange={onChange}
onSearchChange={setSearch}
onLoadMore={() => void fetchNextPage()}
hasNextPage={hasNextPage}
isFetchingNextPage={isFetchingNextPage}
isLoading={isPending}
placeholder="Search users by email"
emptyText="No matching users"
errorText={isError ? "Users could not be loaded. Refresh the page to retry." : undefined}
/>
);
};
const RouterField: React.FC<{
options: SearchSelectOption[];
routerNames: string[];
onChange: (names: string[]) => void;
direction: ShadowEvalDirection;
}> = ({ options, routerNames, onChange, direction }) => (
<Field label="Auto-routers">
<MultiSelect
options={options}
value={routerNames}
onValueChange={onChange}
placeholder="Select up to 4 auto-routers"
emptyText="No auto-routers configured"
/>
{routerNames.length > MAX_ROUTERS && (
<p className="text-xs text-destructive">Pick at most {MAX_ROUTERS} auto-routers</p>
)}
{direction === "reverse" && routerNames.length > 1 && (
<p className="text-xs text-destructive">A regression check compares one router to its baseline</p>
)}
{direction === "forward" && routerNames.length > 1 && (
<p className="text-xs text-muted-foreground">
Every router sees the same sampled requests, judged against the same live responses
</p>
)}
</Field>
);
interface StartFormValidityInputs {
accessToken: string | null | undefined;
apiKeyIds: string[];
teamIds: string[];
userIds: string[];
routerNames: string[];
direction: ShadowEvalDirection;
baselineModel: string;
judgeModel: string;
percentage: string;
maxBudget: string;
}
const startFormValidity = (inputs: StartFormValidityInputs) => {
const parsedPct = Number.parseFloat(inputs.percentage);
const percentageValid = parsedPct >= 0.1 && parsedPct <= 100;
const parsedMaxBudget = Number.parseFloat(inputs.maxBudget);
const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000;
const baselinePicked = inputs.direction === "forward" || inputs.baselineModel !== "";
const targetsPicked = inputs.apiKeyIds.length + inputs.teamIds.length + inputs.userIds.length > 0;
const routerCountValid = inputs.routerNames.length >= 1 && inputs.routerNames.length <= MAX_ROUTERS;
const routersMatchDirection = inputs.direction === "forward" || inputs.routerNames.length === 1;
const routersValid = routerCountValid && routersMatchDirection;
const modelsPicked = routersValid && inputs.judgeModel !== "" && baselinePicked;
const filled = targetsPicked && modelsPicked;
const boundsValid = percentageValid && maxBudgetValid;
const valid = Boolean(inputs.accessToken) && filled && boundsValid;
return { parsedPct, parsedMaxBudget, percentageValid, maxBudgetValid, valid };
};
interface StartBodyInputs {
apiKeyIds: string[];
teamIds: string[];
userIds: string[];
routerNames: string[];
direction: ShadowEvalDirection;
baselineModel: string;
shadowPercentage: number;
durationDays: number;
maxBudget: number;
judgeModel: string;
}
const buildStartBody = (inputs: StartBodyInputs) => ({
api_key_ids: inputs.apiKeyIds,
team_ids: inputs.teamIds,
user_ids: inputs.userIds,
router_names: inputs.routerNames,
direction: inputs.direction,
...(inputs.direction === "reverse" ? { baseline_model: inputs.baselineModel } : {}),
shadow_percentage: inputs.shadowPercentage,
duration_days: inputs.durationDays,
max_budget: inputs.maxBudget,
judge_model: inputs.judgeModel,
});
export const StartForm: React.FC = () => {
const { accessToken } = useAuthorized();
const [apiKeyIds, setApiKeyIds] = useState<string[]>([]);
const [teamIds, setTeamIds] = useState<string[]>([]);
const [userIds, setUserIds] = useState<string[]>([]);
const [routerNames, setRouterNames] = useState<string[]>([]);
const [direction, setDirection] = useState<ShadowEvalDirection>("forward");
const [baselineModel, setBaselineModel] = useState("");
const [percentage, setPercentage] = useState("10");
const [durationDays, setDurationDays] = useState("7");
const [judgeModel, setJudgeModel] = useState("");
const [maxBudget, setMaxBudget] = useState("10");
const { data: autoRouters } = useAutoRouters();
const judgeModelOptions = useJudgeModelOptions();
const baselineModelOptions = useBaselineModelOptions();
const start = useStartShadowEval();
const routerOptions = useMemo<SearchSelectOption[]>(() => {
const names = new Set(
(autoRouters ?? []).map((deployment) => deployment.model_name).filter((name): name is string => Boolean(name)),
);
return [...names].toSorted().map((name) => ({ label: name, value: name }));
}, [autoRouters]);
const validityInputs: StartFormValidityInputs = {
accessToken,
apiKeyIds,
teamIds,
userIds,
routerNames,
direction,
baselineModel,
judgeModel,
percentage,
maxBudget,
};
const { parsedPct, parsedMaxBudget, percentageValid, maxBudgetValid, valid } = startFormValidity(validityInputs);
const handleStart = () => {
const bodyInputs: StartBodyInputs = {
apiKeyIds,
teamIds,
userIds,
routerNames,
direction,
baselineModel,
shadowPercentage: parsedPct,
durationDays: Number.parseInt(durationDays, 10),
maxBudget: parsedMaxBudget,
judgeModel,
};
start.mutate(buildStartBody(bodyInputs));
};
return (
<Card size="sm">
<CardHeader>
<CardTitle className="text-sm font-medium text-foreground">Start a shadow eval</CardTitle>
<p className="text-xs text-muted-foreground">{START_FORM_DESCRIPTION[direction]}</p>
</CardHeader>
<CardContent className="space-y-3">
<div className="grid gap-3 sm:grid-cols-3">
<Field label="Direction">
<Select
value={direction}
onValueChange={(v: string | null) => setDirection(v === "reverse" ? "reverse" : "forward")}
>
<SelectTrigger className="w-full">
<SelectValue>{DIRECTION_OPTIONS.find((o) => o.value === direction)?.label}</SelectValue>
</SelectTrigger>
<SelectContent>
{DIRECTION_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
</Field>
<Field label="Keys to shadow" htmlFor="shadow-eval-key">
<KeySelect value={apiKeyIds} onChange={setApiKeyIds} />
</Field>
<Field label="Teams to shadow">
<TeamMultiSelect value={teamIds} onChange={setTeamIds} placeholder="Search teams by alias" />
</Field>
<Field label="Users to shadow" htmlFor="shadow-eval-user">
<UserSelect value={userIds} onChange={setUserIds} />
</Field>
<RouterField
options={routerOptions}
routerNames={routerNames}
onChange={setRouterNames}
direction={direction}
/>
<Field label="Traffic sampled" htmlFor="shadow-eval-pct">
<div className="flex items-center gap-2">
<Input
id="shadow-eval-pct"
type="number"
min={0.1}
max={100}
step={0.1}
className="w-24"
value={percentage}
onChange={(e) => setPercentage(e.target.value)}
/>
<span className="text-sm text-muted-foreground">% of traffic</span>
</div>
<div>
{percentage.trim() !== "" && !percentageValid && (
<p className="text-xs text-destructive">Enter a value from 0.1 to 100</p>
)}
</div>
</Field>
<Field label="Duration">
<Select value={durationDays} onValueChange={(v: string | null) => setDurationDays(v ?? "7")}>
<SelectTrigger className="w-full">
<SelectValue>{DURATION_OPTIONS.find((o) => o.value === durationDays)?.label}</SelectValue>
</SelectTrigger>
<SelectContent>
{DURATION_OPTIONS.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
</Field>
<Field label="Spend budget">
<div className="flex items-center gap-2">
<span className="text-sm text-muted-foreground">$</span>
<Input
type="number"
min={0.01}
max={10000}
step={0.01}
className="w-24"
value={maxBudget}
onChange={(e) => setMaxBudget(e.target.value)}
/>
<span className="text-sm text-muted-foreground">max shadow + judge spend, per target</span>
</div>
{maxBudget.trim() !== "" && !maxBudgetValid && (
<p className="text-xs text-destructive">Enter a value from 0.01 to 10000</p>
)}
</Field>
{direction === "reverse" && (
<Field label="Baseline model">
<SearchSelect
options={baselineModelOptions}
value={baselineModel}
onValueChange={setBaselineModel}
placeholder="Select a baseline model"
emptyText="No chat models available"
/>
</Field>
)}
<Field label="Judge model" className="sm:col-span-2">
<SearchSelect
options={judgeModelOptions}
value={judgeModel}
onValueChange={setJudgeModel}
placeholder="Select a judge model"
emptyText="No chat models available"
/>
</Field>
</div>
<Button disabled={!valid || start.isPending} onClick={handleStart}>
{start.isPending ? "Starting..." : "Start shadow eval"}
</Button>
</CardContent>
</Card>
);
};

35
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -35331,8 +35331,17 @@ export interface components {
last_error?: string | null;
/** @description Stratified verdicts; detail endpoint only */
results?: components["schemas"]["ShadowEvalResult"] | null;
/** Router Name */
router_name: string;
/**
* Router Name
* @description The first router, kept for callers that predate router_names; derived so the
* two fields can never disagree.
*/
readonly router_name: string;
/**
* Router Names
* @description Every auto-router this job runs as a shadow arm. Multi-router jobs sample one slice of traffic and judge every arm against the same real responses
*/
router_names: string[];
/** Shadow Percentage */
shadow_percentage: number;
/**
@ -35420,6 +35429,12 @@ export interface components {
* @description Sliced by the model that served the real arm: the keys' incumbent models in forward mode, and in reverse the models the router itself picked
*/
by_current_model: components["schemas"]["ShadowEvalSlice"][];
/**
* By Router
* @description One slice per router arm, grouped on the router name. Every arm of a multi-router job is judged against the same real responses over the same sampled requests, so these slices compare routers head-to-head: like-for-like win rates and spends on identical traffic. Verdicts from before arm stamping existed count toward the job's own router
* @default []
*/
by_router: components["schemas"]["ShadowEvalSlice"][];
/** By Tier */
by_tier: components["schemas"]["ShadowEvalSlice"][];
/**
@ -35433,13 +35448,13 @@ export interface components {
overall_tie_rate_pct: number;
/**
* Sampled Real Spend
* @description USD the real arm billed across all judged turns, cache-served turns excluded
* @description USD the real arm billed across all judged turns, cache-served turns excluded. A judged turn is one (request, router arm) verdict, so a multi-router job counts the real response once per arm it was judged against; per-router comparisons read by_router
* @default 0
*/
sampled_real_spend: number;
/**
* Sampled Shadow Spend
* @description USD the shadow arm billed across the same turns, judge excluded, like for like
* @description USD the shadow arms billed across the same turns, judge excluded, like for like
* @default 0
*/
sampled_shadow_spend: number;
@ -35733,15 +35748,21 @@ export interface components {
judge_model: string;
/**
* Max Budget
* @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window
* @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window. Every router arm draws from the same per-target budget, so a multi-router job reaches it proportionally sooner
* @default 10
*/
max_budget: number;
/**
* Router Name
* @description The auto-router under evaluation, in either direction
* @description The auto-router under evaluation, in either direction: the single-router spelling of router_names. Provide exactly one of the two fields
*/
router_name: string;
router_name?: string | null;
/**
* Router Names
* @description The auto-routers under evaluation, at most 4. Every sampled request runs through every router listed and each arm is judged independently against the same real response, so routers compare head-to-head on identical traffic. More than one router requires direction 'forward'. After validation this field always carries the full deduplicated set, whichever spelling the caller used
* @default []
*/
router_names: string[];
/**
* Shadow Percentage
* @description Percentage of each target's requests to duplicate through the router