mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
feat(shadow_eval): target teams and users so JWT-auth traffic can be evaluated (#39015)
Shadow eval jobs previously targeted only virtual keys, so deployments on pure JWT auth (which present no key at all) could never sample their traffic. Jobs now carry a typed (target_type, target_id) pair covering keys, teams, and users; sampling matches the identity every request resolves to at auth time, so team and user jobs cover JWT traffic with no client changes. Resolves LIT-6578
This commit is contained in:
tin-berri
GitHub
parent
f93d9b6b67
commit
3829418878
13 changed files with 1181 additions and 391 deletions
|
|
@ -0,0 +1,21 @@
|
|||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'LiteLLM_ShadowEvalJob' AND column_name = 'api_key_id'
|
||||
) THEN
|
||||
ALTER TABLE "LiteLLM_ShadowEvalJob" RENAME COLUMN "api_key_id" TO "target_id";
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "target_type" TEXT NOT NULL DEFAULT 'key';
|
||||
|
||||
DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key_direction";
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_one_active_per_target_direction"
|
||||
ON "LiteLLM_ShadowEvalJob"("target_type", "target_id", "direction") WHERE "stopped_at" IS NULL;
|
||||
|
||||
DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_api_key_id_idx";
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_target_type_target_id_idx"
|
||||
ON "LiteLLM_ShadowEvalJob"("target_type", "target_id");
|
||||
|
|
@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession {
|
|||
model LiteLLM_ShadowEvalJob {
|
||||
id String @id @default(cuid())
|
||||
group_id String // legs of one job share this; the API's job id
|
||||
api_key_id String // hashed virtual key whose traffic this leg shadows
|
||||
target_type String @default("key") // key | team | user
|
||||
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
|
||||
router_name String // the auto-router under evaluation, in either direction
|
||||
direction String @default("forward") // forward | reverse
|
||||
baseline_model String? // reverse only: the fixed model the router is judged against
|
||||
judge_model String
|
||||
shadow_percentage Float
|
||||
max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
|
||||
max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
|
||||
max_budget Float? // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
|
||||
created_at DateTime @default(now())
|
||||
created_by String?
|
||||
ends_at DateTime
|
||||
|
|
@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
stopped_by String? // operator who stopped it early; null when it ended on its own
|
||||
|
||||
@@index([group_id])
|
||||
@@index([api_key_id])
|
||||
@@index([target_type, target_id])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -592,7 +592,12 @@ _JOBS_CACHE_KEY: Final = "shadow_eval:active_jobs"
|
|||
|
||||
|
||||
class ShadowEvalLogger(CustomLogger):
|
||||
"""Fires blind pairwise shadow evaluations for keys with an active shadow-eval job."""
|
||||
"""Fires blind pairwise shadow evaluations for targets with an active shadow-eval job.
|
||||
|
||||
A job targets a virtual key, a team, or a user; a request qualifies for a job when
|
||||
any of its resolved identities (key hash, team id, user id) matches the job's
|
||||
target, so team and user jobs cover JWT-authenticated traffic, which carries no
|
||||
key hash at all."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -617,10 +622,10 @@ class ShadowEvalLogger(CustomLogger):
|
|||
# generation; the refill absorbs written rows and resets.
|
||||
self._job_starts: dict[str, int] = {} # mutable-ok: per-generation counter
|
||||
|
||||
async def _active_jobs(self) -> Mapping[str, tuple[ActiveShadowEvalJob, ...]]:
|
||||
"""Active jobs by api_key_id, cache-first. A key holds at most one job per
|
||||
direction, so the value is a collection. A DB fault returns empty without
|
||||
caching, so sampling pauses for that request and the next one retries."""
|
||||
async def _active_jobs(self) -> Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]:
|
||||
"""Active jobs by (target_type, target_id), cache-first. A target holds at most
|
||||
one job per direction, so the value is a collection. A DB fault returns empty
|
||||
without caching, so sampling pauses for that request and the next one retries."""
|
||||
cached: Final = await self._jobs_cache.async_get_cache(_JOBS_CACHE_KEY)
|
||||
if cached is not None:
|
||||
return cached # pyright: ignore[reportReturnType] # cache stores exactly this mapping shape
|
||||
|
|
@ -652,10 +657,10 @@ class ShadowEvalLogger(CustomLogger):
|
|||
)
|
||||
for row in grouped or []
|
||||
}
|
||||
by_key: Final = tuple(
|
||||
by_target: Final = tuple(
|
||||
sorted(
|
||||
(
|
||||
(str(record.api_key_id), job)
|
||||
((str(record.target_type), str(record.target_id)), job)
|
||||
for record in records or []
|
||||
if (job := _as_active_job(record, *attempt_stats.get(str(record.id), (0, 0.0)))) is not None
|
||||
),
|
||||
|
|
@ -663,7 +668,7 @@ class ShadowEvalLogger(CustomLogger):
|
|||
)
|
||||
)
|
||||
jobs: Final = MappingProxyType(
|
||||
{key: tuple(job for _, job in group) for key, group in groupby(by_key, key=itemgetter(0))}
|
||||
{target: tuple(job for _, job in group) for target, group in groupby(by_target, key=itemgetter(0))}
|
||||
)
|
||||
await self._jobs_cache.async_set_cache(_JOBS_CACHE_KEY, jobs)
|
||||
self._job_starts = {} # rebind-ok: new generation, counts absorbed into the fill
|
||||
|
|
@ -720,8 +725,18 @@ class ShadowEvalLogger(CustomLogger):
|
|||
if should_redact_message_logging(dict(kwargs)): # mutable-ok: predicate takes a plain dict
|
||||
return
|
||||
metadata: Final = payload.get("metadata") or _EMPTY_METADATA
|
||||
api_key_hash: Final = metadata.get("user_api_key_hash")
|
||||
if not api_key_hash:
|
||||
# Each identity the request resolved to is a candidate target; JWT-auth
|
||||
# requests carry no key hash but do carry a team and user.
|
||||
targets: Final = tuple(
|
||||
(target_type, str(value))
|
||||
for target_type, value in (
|
||||
("key", metadata.get("user_api_key_hash")),
|
||||
("team", metadata.get("user_api_key_team_id")),
|
||||
("user", metadata.get("user_api_key_user_id")),
|
||||
)
|
||||
if value
|
||||
)
|
||||
if not targets:
|
||||
return
|
||||
request_id: Final = payload.get("id") or ""
|
||||
if not request_id:
|
||||
|
|
@ -731,8 +746,11 @@ class ShadowEvalLogger(CustomLogger):
|
|||
return # only surfaces this table can normalize are comparable; unknown types fail closed
|
||||
if ops.wire_params and _request_mutating_guardrail_ran(request_metadata):
|
||||
return # the wire-body snapshot predates the rewrite; replaying it would resurrect stripped content
|
||||
active_jobs: Final = await self._active_jobs()
|
||||
eligible: Final = self._sampled_jobs(
|
||||
(await self._active_jobs()).get(str(api_key_hash), ()), request_metadata, request_id
|
||||
tuple(job for target in targets for job in active_jobs.get(target, ())),
|
||||
request_metadata,
|
||||
request_id,
|
||||
)
|
||||
if not eligible:
|
||||
return
|
||||
|
|
@ -1056,7 +1074,7 @@ class ShadowEvalLogger(CustomLogger):
|
|||
)
|
||||
|
||||
|
||||
_EMPTY_JOBS: Final[Mapping[str, tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({})
|
||||
_EMPTY_JOBS: Final[Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _default_prisma_provider() -> "PrismaClient | None":
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ POST /auto_router/validate_complexity_router_config - Dry-run the complexity-rou
|
|||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import groupby
|
||||
from itertools import chain, groupby
|
||||
from operator import attrgetter
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Protocol
|
||||
|
|
@ -58,10 +58,11 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
|
|||
ComplexityRouterConfigValidationResponse,
|
||||
RequestComplexityRouterConfig,
|
||||
ShadowEvalDirection,
|
||||
ShadowEvalJobKeyResponse,
|
||||
ShadowEvalJobResponse,
|
||||
ShadowEvalJobTargetResponse,
|
||||
ShadowEvalResult,
|
||||
ShadowEvalSlice,
|
||||
ShadowEvalTargetType,
|
||||
StartShadowEvalRequest,
|
||||
)
|
||||
|
||||
|
|
@ -104,10 +105,43 @@ class _VerificationTokenTable(Protocol):
|
|||
async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_VerificationTokenRow]: ...
|
||||
|
||||
|
||||
class _TeamRow(Protocol):
|
||||
@property
|
||||
def team_id(self) -> str: ...
|
||||
|
||||
@property
|
||||
def team_alias(self) -> str | None: ...
|
||||
|
||||
|
||||
class _TeamRowsTable(Protocol):
|
||||
async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_TeamRow]: ...
|
||||
|
||||
|
||||
class _UserRow(Protocol):
|
||||
@property
|
||||
def user_id(self) -> str: ...
|
||||
|
||||
@property
|
||||
def user_email(self) -> str | None: ...
|
||||
|
||||
|
||||
class _UserRowsTable(Protocol):
|
||||
async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_UserRow]: ...
|
||||
|
||||
|
||||
class _ShadowEvalJobRow(Protocol):
|
||||
@property
|
||||
def id(self) -> str: ...
|
||||
|
||||
@property
|
||||
def group_id(self) -> str: ...
|
||||
|
||||
@property
|
||||
def target_type(self) -> str: ...
|
||||
|
||||
@property
|
||||
def target_id(self) -> str: ...
|
||||
|
||||
|
||||
class _ShadowEvalJobTable(Protocol):
|
||||
async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ShadowEvalJobRow]: ...
|
||||
|
|
@ -138,6 +172,14 @@ def _verification_tokens(prisma_client: "PrismaClient") -> _VerificationTokenTab
|
|||
return prisma_client.db.litellm_verificationtoken
|
||||
|
||||
|
||||
def _team_rows(prisma_client: "PrismaClient") -> _TeamRowsTable:
|
||||
return prisma_client.db.litellm_teamtable
|
||||
|
||||
|
||||
def _user_rows(prisma_client: "PrismaClient") -> _UserRowsTable:
|
||||
return prisma_client.db.litellm_usertable
|
||||
|
||||
|
||||
def _shadow_eval_jobs(prisma_client: "PrismaClient") -> _ShadowEvalJobTable:
|
||||
return prisma_client.db.litellm_shadowevaljob
|
||||
|
||||
|
|
@ -836,7 +878,7 @@ def _validate_judge_is_not_a_candidate(
|
|||
|
||||
|
||||
def _is_unique_violation(error: Exception) -> bool:
|
||||
"""Whether a Prisma create failed on a unique index. One active job per key and
|
||||
"""Whether a Prisma create failed on a unique index. One active job per target and
|
||||
direction lives in a partial unique index (raw SQL in the migration; schema.prisma
|
||||
cannot express partial indexes), so the read-then-create check above it is advisory:
|
||||
two concurrent starts pass the read, and the loser must surface as the same 409
|
||||
|
|
@ -885,7 +927,7 @@ _ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT
|
|||
# direction, and mid-deploy rows from old pods price as judge-only until the deploy ends).
|
||||
_SWEEP_FINISHED_JOBS_SQL: Final = """
|
||||
UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = (NOW() AT TIME ZONE 'utc')
|
||||
WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL
|
||||
WHERE j.target_type = $2 AND j.target_id = ANY($1::text[]) AND j.stopped_at IS NULL
|
||||
AND (
|
||||
j.ends_at <= (NOW() AT TIME ZONE 'utc')
|
||||
OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns
|
||||
|
|
@ -966,10 +1008,10 @@ WHERE group_id IN (
|
|||
)
|
||||
"""
|
||||
|
||||
_LIST_LEGS_BY_KEY_SQL: Final = """
|
||||
_LIST_LEGS_BY_TARGET_SQL: Final = """
|
||||
SELECT * FROM "LiteLLM_ShadowEvalJob"
|
||||
WHERE group_id IN (
|
||||
SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE api_key_id = $2
|
||||
SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE target_type = $2 AND target_id = $3
|
||||
GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int
|
||||
)
|
||||
"""
|
||||
|
|
@ -1007,15 +1049,16 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]:
|
|||
|
||||
class _LegRow(BaseModel):
|
||||
"""One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is
|
||||
one key's leg of a job; the legs of a job share group_id and identical config, written
|
||||
together by one create_many. The API's job id is the group id, so leg ids never leave
|
||||
the server (attempts reference them internally)."""
|
||||
one target's leg of a job; the legs of a job share group_id and identical config,
|
||||
written together by one create_many. The API's job id is the group id, so leg ids
|
||||
never leave the server (attempts reference them internally)."""
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: str
|
||||
group_id: str
|
||||
api_key_id: str
|
||||
target_type: ShadowEvalTargetType
|
||||
target_id: str
|
||||
router_name: str
|
||||
direction: ShadowEvalDirection
|
||||
baseline_model: str | None = None
|
||||
|
|
@ -1068,16 +1111,17 @@ def _group_response(
|
|||
first: Final = legs[0]
|
||||
return ShadowEvalJobResponse(
|
||||
job_id=group_id,
|
||||
keys=tuple(
|
||||
ShadowEvalJobKeyResponse(
|
||||
api_key_id=leg.api_key_id,
|
||||
targets=tuple(
|
||||
ShadowEvalJobTargetResponse(
|
||||
target_type=leg.target_type,
|
||||
target_id=leg.target_id,
|
||||
max_turns=leg.max_turns,
|
||||
max_budget=leg.max_budget,
|
||||
stopped_at=leg.stopped_at,
|
||||
attempt_count=stats.attempt_count if (stats := attempt_counts.get(leg.id)) else 0,
|
||||
spend=round(stats.spend, 6) if stats else 0.0,
|
||||
)
|
||||
for leg in sorted(legs, key=lambda leg: leg.api_key_id)
|
||||
for leg in sorted(legs, key=lambda leg: (leg.target_type, leg.target_id))
|
||||
),
|
||||
router_name=first.router_name,
|
||||
direction=first.direction,
|
||||
|
|
@ -1090,34 +1134,85 @@ def _group_response(
|
|||
)
|
||||
|
||||
|
||||
_NO_KEY_LABELS: Final[tuple[str | None, str | None]] = (None, None)
|
||||
_NO_TARGET_LABELS: Final[tuple[str | None, str | None]] = (None, None)
|
||||
|
||||
|
||||
async def _with_key_labels(
|
||||
def _target_labels(
|
||||
key_rows: Sequence[_VerificationTokenRow],
|
||||
team_rows: Sequence[_TeamRow],
|
||||
user_rows: Sequence[_UserRow],
|
||||
) -> Mapping[tuple[str, str], tuple[str | None, str | None]]:
|
||||
"""Display labels by (target_type, target_id): a key's (alias, masked name), a
|
||||
team's (alias, None), a user's (email, None)."""
|
||||
return MappingProxyType(
|
||||
{ # mutable-ok: MappingProxyType needs a dict to wrap
|
||||
key: value
|
||||
for key, value in chain(
|
||||
((("key", row.token), (row.key_alias, row.key_name)) for row in key_rows),
|
||||
((("team", row.team_id), (row.team_alias, None)) for row in team_rows),
|
||||
((("user", row.user_id), (row.user_email, None)) for row in user_rows),
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _target_ids_of(responses: Sequence[ShadowEvalJobResponse], target_type: ShadowEvalTargetType) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
frozenset(
|
||||
target.target_id
|
||||
for response in responses
|
||||
for target in response.targets
|
||||
if target.target_type == target_type
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def _with_target_labels(
|
||||
prisma_client: "PrismaClient", responses: Sequence[ShadowEvalJobResponse]
|
||||
) -> tuple[ShadowEvalJobResponse, ...]:
|
||||
"""Resolve every scoped key's hash to its alias and masked name in one batched read,
|
||||
so the UI can say whose traffic a job shadows. Deleted keys resolve to None."""
|
||||
"""Resolve every scoped target's id to a display label in one batched read per kind,
|
||||
so the UI can say whose traffic a job shadows: a key's alias and masked name, a
|
||||
team's alias, a user's email. Deleted targets resolve to None."""
|
||||
if not responses:
|
||||
return ()
|
||||
tokens: Final = sorted(frozenset(key.api_key_id for response in responses for key in response.keys))
|
||||
key_rows: Final = await _verification_tokens(prisma_client).find_many(
|
||||
where={"token": {"in": tokens}} # mutable-ok: Prisma filter
|
||||
tokens: Final = _target_ids_of(responses, "key")
|
||||
team_ids: Final = _target_ids_of(responses, "team")
|
||||
user_ids: Final = _target_ids_of(responses, "user")
|
||||
key_rows: Final = (
|
||||
await _verification_tokens(prisma_client).find_many(
|
||||
where={"token": {"in": list(tokens)}} # mutable-ok: Prisma filter
|
||||
)
|
||||
if tokens
|
||||
else ()
|
||||
)
|
||||
labels: Final[Mapping[str, tuple[str | None, str | None]]] = {
|
||||
row.token: (row.key_alias, row.key_name) for row in key_rows or ()
|
||||
}
|
||||
team_rows: Final = (
|
||||
await _team_rows(prisma_client).find_many(
|
||||
where={"team_id": {"in": list(team_ids)}} # mutable-ok: Prisma filter
|
||||
)
|
||||
if team_ids
|
||||
else ()
|
||||
)
|
||||
user_rows: Final = (
|
||||
await _user_rows(prisma_client).find_many(
|
||||
where={"user_id": {"in": list(user_ids)}} # mutable-ok: Prisma filter
|
||||
)
|
||||
if user_ids
|
||||
else ()
|
||||
)
|
||||
labels: Final = _target_labels(key_rows or (), team_rows or (), user_rows or ())
|
||||
return tuple(
|
||||
response.model_copy(
|
||||
update={ # mutable-ok: pydantic update payload
|
||||
"keys": tuple(
|
||||
key.model_copy(
|
||||
"targets": tuple(
|
||||
target.model_copy(
|
||||
update={ # mutable-ok: pydantic update payload
|
||||
"key_alias": labels.get(key.api_key_id, _NO_KEY_LABELS)[0],
|
||||
"key_name": labels.get(key.api_key_id, _NO_KEY_LABELS)[1],
|
||||
"target_alias": labels.get((target.target_type, target.target_id), _NO_TARGET_LABELS)[0],
|
||||
"key_name": labels.get((target.target_type, target.target_id), _NO_TARGET_LABELS)[1],
|
||||
}
|
||||
)
|
||||
for key in response.keys
|
||||
for target in response.targets
|
||||
)
|
||||
}
|
||||
)
|
||||
|
|
@ -1125,29 +1220,37 @@ async def _with_key_labels(
|
|||
)
|
||||
|
||||
|
||||
async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> ShadowEvalResult | None:
|
||||
"""All three stratifications of one job's verdicts. Tier answers "where does the router
|
||||
do well"; the model stratification groups by whichever model served the real arm, so it
|
||||
answers "which of the models these keys use today would the router beat" forward, and
|
||||
"for the turns the router sent to X, did X beat the baseline" in reverse; key answers
|
||||
"which key's traffic does the router suit". Reads are bounded by the job's own attempts
|
||||
(<= the sum of its keys' max_turns) via the job_id index."""
|
||||
async def _shadow_eval_results(
|
||||
prisma_client: "PrismaClient", legs: Sequence[_LegRow]
|
||||
) -> tuple[ShadowEvalResult | None, Mapping[tuple[str, str], ShadowEvalSlice]]:
|
||||
"""One job's stratified verdicts, plus each target's own slice keyed by the
|
||||
(target_type, target_id) pair so a key, team, and user sharing an id can never
|
||||
collapse into one entry. Tier answers "where does the router do well"; the model
|
||||
stratification groups by whichever model served the real arm, so it answers "which
|
||||
of the models these targets use today would the router beat" forward, and "for the
|
||||
turns the router sent to X, did X beat the baseline" in reverse; the per-target
|
||||
slices answer "which target's traffic does the router suit". Reads are bounded by
|
||||
the job's own attempts (<= the sum of its targets' max_turns) via the job_id index."""
|
||||
leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param
|
||||
by_tier: Final = _ATTEMPT_AGG_ROWS.validate_python(
|
||||
await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, leg_ids) or ()
|
||||
)
|
||||
if not by_tier:
|
||||
return None
|
||||
return None, MappingProxyType({})
|
||||
by_model: Final = _ATTEMPT_AGG_ROWS.validate_python(
|
||||
await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, leg_ids) or ()
|
||||
)
|
||||
key_by_leg: Final = MappingProxyType({leg.id: leg.api_key_id for leg in legs})
|
||||
target_by_leg: Final = MappingProxyType({leg.id: (leg.target_type, leg.target_id) for leg in legs})
|
||||
by_leg: Final = _ATTEMPT_AGG_ROWS.validate_python(
|
||||
await _query_raw(prisma_client, _ATTEMPT_AGG_BY_LEG_SQL, leg_ids) or ()
|
||||
)
|
||||
by_key: Final = tuple(
|
||||
row.model_copy(update={"grp": key_by_leg[row.grp]}) # mutable-ok: pydantic update payload
|
||||
for row in by_leg
|
||||
verdicts_by_target: Final[Mapping[tuple[str, str], ShadowEvalSlice]] = MappingProxyType(
|
||||
{ # mutable-ok: MappingProxyType needs a dict to wrap
|
||||
target_by_leg[slice.group]: slice.model_copy(
|
||||
update={"group": target_by_leg[slice.group][1]} # mutable-ok: pydantic update payload
|
||||
)
|
||||
for slice in _slices(by_leg)
|
||||
}
|
||||
)
|
||||
total_turns: Final = sum(r.turn_count for r in by_tier)
|
||||
funnel_rows: Final = await _query_raw(prisma_client, _FUNNEL_TOTALS_SQL, leg_ids)
|
||||
|
|
@ -1155,10 +1258,9 @@ async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_Le
|
|||
# Coverage only when EVERY leg has a funnel row: a partial seed (one leg's insert
|
||||
# failed) must read as unknown, not as job-level counts missing a leg's traffic.
|
||||
funnel: Final = counted if counted is not None and counted.legs_with_rows == len(leg_ids) else None
|
||||
return ShadowEvalResult(
|
||||
result: Final = ShadowEvalResult(
|
||||
by_tier=_slices(by_tier),
|
||||
by_current_model=_slices(by_model),
|
||||
by_key=_slices(by_key),
|
||||
overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns),
|
||||
overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns),
|
||||
sampled_real_spend=sum(r.real_spend for r in by_tier),
|
||||
|
|
@ -1168,6 +1270,7 @@ async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_Le
|
|||
shed_count=funnel.shed if funnel is not None else None,
|
||||
withheld_count=funnel.withheld if funnel is not None else None,
|
||||
)
|
||||
return result, verdicts_by_target
|
||||
|
||||
|
||||
@router.post(
|
||||
|
|
@ -1182,22 +1285,29 @@ async def start_shadow_eval(
|
|||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> ShadowEvalJobResponse:
|
||||
"""
|
||||
Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against
|
||||
a second arm, judge the two responses blind, and stratify win rates by tier, by the model
|
||||
that served the real arm, and by key.
|
||||
Start a shadow eval: duplicate a sampled slice of one or more targets' live traffic
|
||||
against a second arm, judge the two responses blind, and stratify win rates by tier,
|
||||
by the model that served the real arm, and by target.
|
||||
|
||||
A forward job answers whether the keys should adopt router_name: it samples the requests
|
||||
the router did not serve and duplicates them through it. A reverse job answers whether a
|
||||
key already on the router still gains from it: it samples the requests the router did
|
||||
serve and duplicates them against baseline_model. A key can hold one active job per
|
||||
direction, so both questions can run at once.
|
||||
A target is a virtual key, a team, or a user. Team and user targets match on the
|
||||
identity every request resolves to at auth time, so they cover JWT-authenticated
|
||||
traffic, which presents no virtual key; a user target samples that user's traffic
|
||||
across all their teams, whether it arrives on a JWT or a key they own.
|
||||
|
||||
Shadow responses are never served to users. Each key samples until its recorded eval
|
||||
spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's
|
||||
window ends, or the job is stopped, so one key running out of budget does not end
|
||||
sampling for the others; sampling changes propagate to pods within about 10 seconds.
|
||||
Shadow and judge calls bill to the shadowed key but are excluded from request counts
|
||||
and auto-router adoption metrics.
|
||||
A forward job answers whether the targets should adopt router_name: it samples the
|
||||
requests the router did not serve and duplicates them through it. A reverse job
|
||||
answers whether a target already on the router still gains from it: it samples the
|
||||
requests the router did serve and duplicates them against baseline_model. A target
|
||||
can hold one active job per direction, so both questions can run at once, and a
|
||||
request matching several jobs' targets (say its key and its team) is sampled by
|
||||
each, separately budgeted.
|
||||
|
||||
Shadow responses are never served to users. Each target samples until its recorded
|
||||
eval spend, the shadow and judge calls' own cost, reaches max_budget dollars, the
|
||||
job's window ends, or the job is stopped, so one target running out of budget does
|
||||
not end sampling for the others; sampling changes propagate to pods within about 10
|
||||
seconds. Shadow and judge calls bill to the sampled request's own identity but are
|
||||
excluded from request counts and auto-router adoption metrics.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router, prisma_client
|
||||
|
||||
|
|
@ -1206,35 +1316,88 @@ async def start_shadow_eval(
|
|||
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
|
||||
if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name):
|
||||
raise HTTPException(status_code=400, detail=f"'{data.router_name}' is not a configured auto-router")
|
||||
token_rows: Final = await _verification_tokens(prisma_client).find_many(
|
||||
where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter
|
||||
)
|
||||
unknown: Final = tuple(sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ())))
|
||||
if unknown:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"api_key_ids not on this proxy: {', '.join(unknown)}; pass each key's token hash, "
|
||||
"the value the key list and key info endpoints report"
|
||||
),
|
||||
token_rows: Final = (
|
||||
await _verification_tokens(prisma_client).find_many(
|
||||
where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter
|
||||
)
|
||||
if data.api_key_ids
|
||||
else ()
|
||||
)
|
||||
team_rows: Final = (
|
||||
await _team_rows(prisma_client).find_many(
|
||||
where={"team_id": {"in": list(data.team_ids)}} # mutable-ok: Prisma filter
|
||||
)
|
||||
if data.team_ids
|
||||
else ()
|
||||
)
|
||||
user_rows: Final = (
|
||||
await _user_rows(prisma_client).find_many(
|
||||
where={"user_id": {"in": list(data.user_ids)}} # mutable-ok: Prisma filter
|
||||
)
|
||||
if data.user_ids
|
||||
else ()
|
||||
)
|
||||
unknown_keys: Final = sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ()))
|
||||
unknown_teams: Final = sorted(frozenset(data.team_ids) - frozenset(row.team_id for row in team_rows or ()))
|
||||
unknown_users: Final = sorted(frozenset(data.user_ids) - frozenset(row.user_id for row in user_rows or ()))
|
||||
unknown_parts: Final = tuple(
|
||||
part
|
||||
for part in (
|
||||
(
|
||||
f"api_key_ids not on this proxy: {', '.join(unknown_keys)}; pass each key's token hash, "
|
||||
"the value the key list and key info endpoints report"
|
||||
)
|
||||
if unknown_keys
|
||||
else None,
|
||||
f"team_ids not on this proxy: {', '.join(unknown_teams)}" if unknown_teams else None,
|
||||
f"user_ids not on this proxy: {', '.join(unknown_users)}" if unknown_users else None,
|
||||
)
|
||||
if part is not None
|
||||
)
|
||||
if unknown_parts:
|
||||
raise HTTPException(status_code=400, detail=". ".join(unknown_parts))
|
||||
|
||||
# Every model check below runs once per team the job samples for, since that is the
|
||||
# identity the shadow and judge calls carry and therefore what the router selects on.
|
||||
team_ids: Final = tuple(dict.fromkeys(row.team_id for row in token_rows or ()))
|
||||
# A user target's traffic can span teams, so it validates unscoped (None); each
|
||||
# sampled attempt still resolves the judge under its own request's team at eval time.
|
||||
team_ids: Final = tuple(
|
||||
dict.fromkeys(
|
||||
(
|
||||
*(row.team_id for row in token_rows or ()),
|
||||
*data.team_ids,
|
||||
*((None,) if data.user_ids else ()),
|
||||
)
|
||||
)
|
||||
)
|
||||
_validate_plain_model(llm_router, data.judge_model, "judge_model", team_ids)
|
||||
if data.baseline_model is not None:
|
||||
_validate_plain_model(llm_router, data.baseline_model, "baseline_model", team_ids)
|
||||
_validate_judge_is_not_a_candidate(llm_router, data, team_ids)
|
||||
|
||||
requested_targets: Final[tuple[tuple[ShadowEvalTargetType, str], ...]] = (
|
||||
*(("key", key) for key in data.api_key_ids),
|
||||
*(("team", team) for team in data.team_ids),
|
||||
*(("user", user) for user in data.user_ids),
|
||||
)
|
||||
requested_by_type: Final[tuple[tuple[ShadowEvalTargetType, tuple[str, ...]], ...]] = tuple(
|
||||
(target_type, ids)
|
||||
for target_type, ids in (("key", data.api_key_ids), ("team", data.team_ids), ("user", data.user_ids))
|
||||
if ids
|
||||
)
|
||||
# A job whose window passed or whose budget ran out stopped sampling on its own,
|
||||
# but its legs still hold their slots in the per-key, per-direction partial unique index
|
||||
# until stamped; free them so a new eval can start. Sweeping both directions is deliberate.
|
||||
requested: Final = list(data.api_key_ids) # mutable-ok: query param
|
||||
await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, requested)
|
||||
# but its legs still hold their slots in the per-target, per-direction partial unique
|
||||
# index until stamped; free them so a new eval can start. Sweeping both directions is
|
||||
# deliberate. Sweep and claim filter on exact (target_type, id) pairs so a team id
|
||||
# that happens to equal a key hash never matches the other kind's slot.
|
||||
for target_type, ids in requested_by_type:
|
||||
await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, list(ids), target_type) # mutable-ok: query param
|
||||
claimed: Final = await _shadow_eval_jobs(prisma_client).find_many(
|
||||
where={ # mutable-ok: Prisma filter
|
||||
"api_key_id": {"in": requested}, # mutable-ok: Prisma filter
|
||||
"OR": [ # mutable-ok: Prisma filter
|
||||
{"target_type": target_type, "target_id": {"in": list(ids)}} # mutable-ok: Prisma filter
|
||||
for target_type, ids in requested_by_type
|
||||
],
|
||||
"direction": data.direction,
|
||||
"stopped_at": None,
|
||||
},
|
||||
|
|
@ -1244,7 +1407,7 @@ async def start_shadow_eval(
|
|||
status_code=409,
|
||||
detail=(
|
||||
f"Already in an active {data.direction} shadow eval job: "
|
||||
+ ", ".join(sorted(f"{row.api_key_id} (job {row.group_id})" for row in claimed))
|
||||
+ ", ".join(sorted(f"{row.target_type} {row.target_id} (job {row.group_id})" for row in claimed))
|
||||
+ ". Stop it first."
|
||||
),
|
||||
)
|
||||
|
|
@ -1268,10 +1431,16 @@ async def start_shadow_eval(
|
|||
# Leg ids are minted here rather than by the DB default so the funnel seed below
|
||||
# writes from the same values with no read-back, which a lagging read replica
|
||||
# (DATABASE_URL_READ_REPLICA) could otherwise return empty.
|
||||
leg_ids: Final = tuple(str(uuid4()) for _ in data.api_key_ids)
|
||||
leg_ids: Final = tuple(str(uuid4()) for _ in requested_targets)
|
||||
await _shadow_eval_jobs(prisma_client).create_many(
|
||||
data=[ # mutable-ok: Prisma payload
|
||||
{**shared_config, "id": leg_id, "api_key_id": key} for leg_id, key in zip(leg_ids, data.api_key_ids)
|
||||
{ # mutable-ok: Prisma payload
|
||||
**shared_config,
|
||||
"id": leg_id,
|
||||
"target_type": target_type,
|
||||
"target_id": target_id,
|
||||
} # mutable-ok: Prisma payload
|
||||
for leg_id, (target_type, target_id) in zip(leg_ids, requested_targets)
|
||||
]
|
||||
)
|
||||
except Exception as e:
|
||||
|
|
@ -1280,7 +1449,8 @@ async def start_shadow_eval(
|
|||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(
|
||||
f"A requested key was claimed by another {data.direction} shadow eval job concurrently. Stop it first."
|
||||
f"A requested target was claimed by another {data.direction} shadow eval job concurrently. "
|
||||
"Stop it first."
|
||||
),
|
||||
) from e
|
||||
# Seed a zero funnel row per leg NOW: a fully covered job never skips a request, so
|
||||
|
|
@ -1293,18 +1463,19 @@ async def start_shadow_eval(
|
|||
)
|
||||
except Exception as seed_err: # noqa: BLE001 # coverage is advisory; the job must still start
|
||||
verbose_proxy_logger.error("shadow_eval: funnel seed failed for job %s: %s", group_id, seed_err)
|
||||
labels: Final = MappingProxyType({row.token: row for row in token_rows})
|
||||
labels: Final = _target_labels(token_rows or (), team_rows or (), user_rows or ())
|
||||
return ShadowEvalJobResponse(
|
||||
job_id=group_id,
|
||||
keys=tuple(
|
||||
ShadowEvalJobKeyResponse(
|
||||
api_key_id=api_key_id,
|
||||
targets=tuple(
|
||||
ShadowEvalJobTargetResponse(
|
||||
target_type=target_type,
|
||||
target_id=target_id,
|
||||
max_turns=SHADOW_EVAL_TURN_VALVE,
|
||||
max_budget=data.max_budget,
|
||||
key_alias=labels[api_key_id].key_alias,
|
||||
key_name=labels[api_key_id].key_name,
|
||||
target_alias=labels.get((target_type, target_id), _NO_TARGET_LABELS)[0],
|
||||
key_name=labels.get((target_type, target_id), _NO_TARGET_LABELS)[1],
|
||||
)
|
||||
for api_key_id in sorted(data.api_key_ids)
|
||||
for target_type, target_id in sorted(requested_targets)
|
||||
),
|
||||
router_name=data.router_name,
|
||||
direction=data.direction,
|
||||
|
|
@ -1324,22 +1495,29 @@ async def start_shadow_eval(
|
|||
)
|
||||
async def list_shadow_eval_jobs(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
api_key_id: Annotated[
|
||||
str | None, Query(description="Filter to jobs that shadow this key, alone or alongside others")
|
||||
target_type: Annotated[
|
||||
ShadowEvalTargetType | None, Query(description="Kind of target to filter on; requires target_id")
|
||||
] = None,
|
||||
target_id: Annotated[
|
||||
str | None, Query(description="Filter to jobs that shadow this target, alone or alongside others")
|
||||
] = None,
|
||||
limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50,
|
||||
) -> tuple[ShadowEvalJobResponse, ...]:
|
||||
"""List shadow eval jobs, newest first, each key with its attempt count so status is
|
||||
accurate. Judged counts, spend, and results ride the detail endpoint only."""
|
||||
"""List shadow eval jobs, newest first, each target with its attempt count so status
|
||||
is accurate. Judged counts, spend, and results ride the detail endpoint only."""
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
_require_admin_viewer(user_api_key_dict, "view shadow evals")
|
||||
if prisma_client is None:
|
||||
raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value)
|
||||
filter_type: Final = target_type if isinstance(target_type, str) else None
|
||||
filter_id: Final = target_id if isinstance(target_id, str) else None
|
||||
if (filter_type is None) != (filter_id is None):
|
||||
raise HTTPException(status_code=400, detail="target_type and target_id filter together; pass both or neither")
|
||||
legs: Final = _LEG_ROWS.validate_python(
|
||||
(
|
||||
await _query_raw(prisma_client, _LIST_LEGS_BY_KEY_SQL, limit, api_key_id)
|
||||
if api_key_id
|
||||
await _query_raw(prisma_client, _LIST_LEGS_BY_TARGET_SQL, limit, filter_type, filter_id)
|
||||
if filter_type and filter_id
|
||||
else await _query_raw(prisma_client, _LIST_LEGS_SQL, limit)
|
||||
)
|
||||
or ()
|
||||
|
|
@ -1354,7 +1532,7 @@ async def list_shadow_eval_jobs(
|
|||
by_group, key=lambda group_id: max(leg.created_at for leg in by_group[group_id]), reverse=True
|
||||
)
|
||||
counts: Final = await _leg_attempt_counts(prisma_client, legs)
|
||||
return await _with_key_labels(
|
||||
return await _with_target_labels(
|
||||
prisma_client, tuple(_group_response(group_id, by_group[group_id], counts) for group_id in newest_first)
|
||||
)
|
||||
|
||||
|
|
@ -1391,16 +1569,25 @@ async def get_shadow_eval_job(
|
|||
where={"job_id": {"in": leg_ids}, "outcome": "error"}, # mutable-ok: Prisma filter
|
||||
order={"created_at": "desc"}, # mutable-ok: Prisma order
|
||||
)
|
||||
labeled: Final = await _with_key_labels(
|
||||
labeled: Final = await _with_target_labels(
|
||||
prisma_client, (_group_response(job_id, legs, await _leg_attempt_counts(prisma_client, legs)),)
|
||||
)
|
||||
results, verdicts_by_target = await _shadow_eval_results(prisma_client, legs)
|
||||
return labeled[0].model_copy(
|
||||
update={ # mutable-ok: pydantic update payload
|
||||
"judged_count": totals[0].judged_count if totals else 0,
|
||||
"error_count": totals[0].error_count if totals else 0,
|
||||
"judge_spend": round(totals[0].judge_spend, 6) if totals else 0.0,
|
||||
"last_error": latest_error.error if latest_error else None,
|
||||
"results": await _shadow_eval_results(prisma_client, legs),
|
||||
"results": results,
|
||||
"targets": tuple(
|
||||
target.model_copy(
|
||||
update={ # mutable-ok: pydantic update payload
|
||||
"verdicts": verdicts_by_target.get((target.target_type, target.target_id))
|
||||
}
|
||||
)
|
||||
for target in labeled[0].targets
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -1415,8 +1602,8 @@ async def stop_shadow_eval_job(
|
|||
job_id: str,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> ShadowEvalJobResponse:
|
||||
"""Stop an active shadow eval job, every key it scopes at once. Attempts are kept;
|
||||
sampling halts within ~10s. Keys that already stopped on their own budget keep the
|
||||
"""Stop an active shadow eval job, every target it scopes at once. Attempts are kept;
|
||||
sampling halts within ~10s. Targets that already stopped on their own budget keep the
|
||||
stopped_at they earned. The statement is the whole state machine: it claims the job
|
||||
only while a leg still samples inside the window with no stop recorded, so a racing
|
||||
operator, a same-instant budget spend, and a repeat stop all read the same 400 with
|
||||
|
|
@ -1443,5 +1630,5 @@ async def stop_shadow_eval_job(
|
|||
current: Final = _group_response(job_id, legs, counts)
|
||||
if claimed == 0:
|
||||
raise HTTPException(status_code=400, detail=f"Job {job_id} is already {current.status}")
|
||||
labeled: Final = await _with_key_labels(prisma_client, (current,))
|
||||
labeled: Final = await _with_target_labels(prisma_client, (current,))
|
||||
return labeled[0]
|
||||
|
|
|
|||
|
|
@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession {
|
|||
model LiteLLM_ShadowEvalJob {
|
||||
id String @id @default(cuid())
|
||||
group_id String // legs of one job share this; the API's job id
|
||||
api_key_id String // hashed virtual key whose traffic this leg shadows
|
||||
target_type String @default("key") // key | team | user
|
||||
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
|
||||
router_name String // the auto-router under evaluation, in either direction
|
||||
direction String @default("forward") // forward | reverse
|
||||
baseline_model String? // reverse only: the fixed model the router is judged against
|
||||
judge_model String
|
||||
shadow_percentage Float
|
||||
max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
|
||||
max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
|
||||
max_budget Float? // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
|
||||
created_at DateTime @default(now())
|
||||
created_by String?
|
||||
ends_at DateTime
|
||||
|
|
@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
stopped_by String? // operator who stopped it early; null when it ended on its own
|
||||
|
||||
@@index([group_id])
|
||||
@@index([api_key_id])
|
||||
@@index([target_type, target_id])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -245,6 +245,8 @@ ShadowEvalStatus: TypeAlias = Literal["running", "completed", "stopped"]
|
|||
|
||||
ShadowEvalDirection: TypeAlias = Literal["forward", "reverse"]
|
||||
|
||||
ShadowEvalTargetType: TypeAlias = Literal["key", "team", "user"]
|
||||
|
||||
DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5"
|
||||
|
||||
# Sample-count ceiling written on every new job: a zero-cost error loop (a shadow arm that
|
||||
|
|
@ -253,16 +255,37 @@ SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000
|
|||
|
||||
|
||||
class StartShadowEvalRequest(BaseModel):
|
||||
"""Start duplicating one or more keys' traffic for blind comparison against an auto-router."""
|
||||
"""Start duplicating one or more targets' traffic for blind comparison against an auto-router.
|
||||
|
||||
A target is a virtual key, a team, or a user; each becomes its own leg with its own
|
||||
budget and stop state. Team and user targets match on the identity every request
|
||||
carries after auth (user_api_key_team_id / user_api_key_user_id), so they cover
|
||||
JWT-authenticated traffic, which presents no virtual key at all."""
|
||||
|
||||
api_key_ids: tuple[str, ...] = Field(
|
||||
min_length=1,
|
||||
default=(),
|
||||
max_length=100,
|
||||
description=(
|
||||
"The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these "
|
||||
"keys' traffic; requests made with any other key are not sampled. Each key carries its own "
|
||||
"max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 "
|
||||
"keys per job, which also bounds every read the job's endpoints make."
|
||||
"Hashed virtual keys whose traffic will be shadowed. Combined with team_ids and user_ids the job "
|
||||
"needs at least one target and at most 100, which also bounds every read the job's endpoints make. "
|
||||
"Each target carries its own max_budget spend budget, so one exhausting its budget leaves the "
|
||||
"others sampling."
|
||||
),
|
||||
)
|
||||
team_ids: tuple[str, ...] = Field(
|
||||
default=(),
|
||||
max_length=100,
|
||||
description=(
|
||||
"Teams whose traffic will be shadowed, matched on the team every authenticated request resolves "
|
||||
"to, so a team's JWT-auth and virtual-key traffic are both sampled"
|
||||
),
|
||||
)
|
||||
user_ids: tuple[str, ...] = Field(
|
||||
default=(),
|
||||
max_length=100,
|
||||
description=(
|
||||
"Users whose traffic will be shadowed, matched on the user every authenticated request resolves "
|
||||
"to across all their teams: JWT requests carrying their subject claim and virtual keys they own"
|
||||
),
|
||||
)
|
||||
router_name: str = Field(description="The auto-router under evaluation, in either direction")
|
||||
|
|
@ -285,7 +308,7 @@ class StartShadowEvalRequest(BaseModel):
|
|||
shadow_percentage: float = Field(
|
||||
ge=0.1,
|
||||
le=100.0,
|
||||
description="Percentage of the key's requests to duplicate through the router",
|
||||
description="Percentage of each target's requests to duplicate through the router",
|
||||
)
|
||||
judge_model: str = Field(
|
||||
default=DEFAULT_SHADOW_EVAL_JUDGE_MODEL,
|
||||
|
|
@ -306,9 +329,9 @@ class StartShadowEvalRequest(BaseModel):
|
|||
ge=0.01,
|
||||
le=10_000,
|
||||
description=(
|
||||
"Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with "
|
||||
"the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval "
|
||||
"spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight "
|
||||
"Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with "
|
||||
"the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval "
|
||||
"spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight "
|
||||
"samples can overshoot the cap by one sampling cache window"
|
||||
),
|
||||
)
|
||||
|
|
@ -319,7 +342,7 @@ class StartShadowEvalRequest(BaseModel):
|
|||
"""Pydantic ignores unknown fields, so a caller still sending max_turns would
|
||||
silently run on the default dollar budget instead of the bound they asked for."""
|
||||
if isinstance(values, Mapping) and "max_turns" in values:
|
||||
raise ValueError("max_turns was replaced by max_budget, the per-key USD cap on the eval's own spend")
|
||||
raise ValueError("max_turns was replaced by max_budget, the per-target USD cap on the eval's own spend")
|
||||
return values
|
||||
|
||||
@field_validator("shadow_percentage")
|
||||
|
|
@ -327,12 +350,21 @@ class StartShadowEvalRequest(BaseModel):
|
|||
def _round_percentage(cls, value: float) -> float:
|
||||
return round(value, 2)
|
||||
|
||||
@field_validator("api_key_ids")
|
||||
@field_validator("api_key_ids", "team_ids", "user_ids")
|
||||
@classmethod
|
||||
def _dedupe_keys(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
||||
"""A key named twice would collide with itself on the one-active-per-(key, direction) index."""
|
||||
def _dedupe_targets(cls, value: tuple[str, ...]) -> tuple[str, ...]:
|
||||
"""A target named twice would collide with itself on the one-active-per-(target, direction) index."""
|
||||
return tuple(dict.fromkeys(value))
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _at_least_one_target_at_most_hundred(self) -> "StartShadowEvalRequest":
|
||||
total: Final = len(self.api_key_ids) + len(self.team_ids) + len(self.user_ids)
|
||||
if total < 1:
|
||||
raise ValueError("at least one target is required: pass api_key_ids, team_ids, or user_ids")
|
||||
if total > 100:
|
||||
raise ValueError("at most 100 targets per job across api_key_ids, team_ids, and user_ids")
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest":
|
||||
if self.direction == "reverse" and self.baseline_model is None:
|
||||
|
|
@ -343,8 +375,9 @@ class StartShadowEvalRequest(BaseModel):
|
|||
|
||||
|
||||
class ShadowEvalSlice(BaseModel):
|
||||
"""Judge outcomes for one slice of a job's verdicts (a router tier, or one of the
|
||||
models that served the real arm)."""
|
||||
"""Judge outcomes for one slice of a job's verdicts: a router tier, one of the
|
||||
models that served the real arm, or one scoped target (embedded on that target's
|
||||
own entry, so slices never need re-joining to a target by id)."""
|
||||
|
||||
group: str
|
||||
turn_count: int
|
||||
|
|
@ -395,12 +428,6 @@ class ShadowEvalResult(BaseModel):
|
|||
"and in reverse the models the router itself picked"
|
||||
)
|
||||
)
|
||||
by_key: tuple[ShadowEvalSlice, ...] = Field(
|
||||
description=(
|
||||
"One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job "
|
||||
"scopes but has not judged a turn for yet are absent rather than reported as zero"
|
||||
),
|
||||
)
|
||||
overall_shadow_win_rate_pct: float
|
||||
overall_tie_rate_pct: float
|
||||
sampled_real_spend: float = Field(
|
||||
|
|
@ -436,27 +463,28 @@ class ShadowEvalResult(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class ShadowEvalJobKeyResponse(BaseModel):
|
||||
"""One key a job shadows, with its own budget and stop state."""
|
||||
class ShadowEvalJobTargetResponse(BaseModel):
|
||||
"""One target a job shadows (a key, team, or user), with its own budget and stop state."""
|
||||
|
||||
api_key_id: str = Field(description="The hashed virtual key whose traffic this entry scopes")
|
||||
target_type: ShadowEvalTargetType = Field(description="What kind of entity this entry scopes")
|
||||
target_id: str = Field(description="The hashed virtual key, team id, or user id whose traffic this entry scopes")
|
||||
max_turns: int = Field(
|
||||
description=(
|
||||
"This key's sample-count ceiling: the whole budget for jobs created before max_budget "
|
||||
"This target's sample-count ceiling: the whole budget for jobs created before max_budget "
|
||||
"existed, and the error-loop safety valve otherwise"
|
||||
)
|
||||
)
|
||||
max_budget: float | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"This key's own USD budget for the eval's shadow and judge spend, independent of its "
|
||||
"This target's own USD budget for the eval's shadow and judge spend, independent of its "
|
||||
"siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds"
|
||||
),
|
||||
)
|
||||
stopped_at: datetime | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"When this key's slot was stamped free, whether its own budget ran out, the window closed, "
|
||||
"When this target's slot was stamped free, whether its own budget ran out, the window closed, "
|
||||
"or an operator stopped the job; status is derived, so a spent budget reads completed even "
|
||||
"while this is still unset"
|
||||
),
|
||||
|
|
@ -464,45 +492,53 @@ class ShadowEvalJobKeyResponse(BaseModel):
|
|||
attempt_count: int | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"This key's sampled attempts so far, judged and errored alike, the same count the sampler "
|
||||
"This target's sampled attempts so far, judged and errored alike, the same count the sampler "
|
||||
"budgets against max_turns; populated on list and detail responses. Frozen at stopped_at "
|
||||
"once the key is stamped, so in-flight attempts landing after a stop never reclassify it"
|
||||
"once the target is stamped, so in-flight attempts landing after a stop never reclassify it"
|
||||
),
|
||||
)
|
||||
spend: float | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets "
|
||||
"This target's recorded shadow plus judge spend in USD, the same figure the sampler budgets "
|
||||
"against max_budget; populated on list and detail responses and frozen at stopped_at "
|
||||
"exactly like attempt_count"
|
||||
),
|
||||
)
|
||||
|
||||
verdicts: "ShadowEvalSlice | None" = Field(
|
||||
default=None,
|
||||
description="This target's own judged-verdict slice; detail endpoint only, None until a turn is judged",
|
||||
)
|
||||
|
||||
@property
|
||||
def budget_spent(self) -> bool:
|
||||
over_spend: Final = self.max_budget is not None and self.spend is not None and self.spend >= self.max_budget
|
||||
return over_spend or (self.attempt_count is not None and self.attempt_count >= self.max_turns)
|
||||
|
||||
key_alias: str | None = Field(
|
||||
target_alias: str | None = Field(
|
||||
default=None,
|
||||
description="Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted",
|
||||
description=(
|
||||
"Display label resolved from the target's own row at read time: the key's alias, the team's "
|
||||
"alias, or the user's email; None when unset or deleted"
|
||||
),
|
||||
)
|
||||
key_name: str | None = Field(
|
||||
default=None,
|
||||
description="Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias",
|
||||
description="Masked display name (sk-...) for key targets, resolved at read time; None for teams and users",
|
||||
)
|
||||
|
||||
|
||||
class ShadowEvalJobResponse(BaseModel):
|
||||
"""A shadow-eval job over one or more keys, each with its own budget and stop state;
|
||||
status is derived from stopped_by, the keys' stop and budget state, and ends_at,
|
||||
"""A shadow-eval job over one or more targets, each with its own budget and stop state;
|
||||
status is derived from stopped_by, the targets' stop and budget state, and ends_at,
|
||||
never stored, so no writer anywhere can produce an inconsistent one. Aggregate
|
||||
fields are populated by the detail endpoint only and stay None on list responses."""
|
||||
|
||||
job_id: str
|
||||
keys: tuple[ShadowEvalJobKeyResponse, ...] = Field(
|
||||
targets: tuple[ShadowEvalJobTargetResponse, ...] = Field(
|
||||
min_length=1,
|
||||
description="The keys whose traffic this job evaluates, and only those keys', each with its own budget",
|
||||
description="The targets whose traffic this job evaluates, and only theirs, each with its own budget",
|
||||
)
|
||||
router_name: str
|
||||
direction: ShadowEvalDirection = "forward"
|
||||
|
|
@ -531,8 +567,8 @@ class ShadowEvalJobResponse(BaseModel):
|
|||
def status(self) -> ShadowEvalStatus:
|
||||
"""Three recorded facts, no history-guessing: a stop is stopped_by (the migration
|
||||
backfills it for every job that displayed stopped when the column arrived, so the
|
||||
pre-column population is closed), completion is the window passing or every key
|
||||
spending its budget, and anything else is running. The all-keys-stamped fallback
|
||||
pre-column population is closed), completion is the window passing or every target
|
||||
spending its budget, and anything else is running. The all-targets-stamped fallback
|
||||
covers only stops written by pre-column pods during a rolling deploy."""
|
||||
if self.stopped_by is not None:
|
||||
return "stopped"
|
||||
|
|
@ -540,8 +576,8 @@ class ShadowEvalJobResponse(BaseModel):
|
|||
self.ends_at if self.ends_at.tzinfo else self.ends_at.replace(tzinfo=timezone.utc)
|
||||
):
|
||||
return "completed"
|
||||
if all(key.budget_spent for key in self.keys):
|
||||
if all(target.budget_spent for target in self.targets):
|
||||
return "completed"
|
||||
if all(key.stopped_at is not None for key in self.keys):
|
||||
if all(target.stopped_at is not None for target in self.targets):
|
||||
return "stopped"
|
||||
return "running"
|
||||
|
|
|
|||
|
|
@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession {
|
|||
model LiteLLM_ShadowEvalJob {
|
||||
id String @id @default(cuid())
|
||||
group_id String // legs of one job share this; the API's job id
|
||||
api_key_id String // hashed virtual key whose traffic this leg shadows
|
||||
target_type String @default("key") // key | team | user
|
||||
target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows
|
||||
router_name String // the auto-router under evaluation, in either direction
|
||||
direction String @default("forward") // forward | reverse
|
||||
baseline_model String? // reverse only: the fixed model the router is judged against
|
||||
judge_model String
|
||||
shadow_percentage Float
|
||||
max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
|
||||
max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
|
||||
max_budget Float? // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
|
||||
created_at DateTime @default(now())
|
||||
created_by String?
|
||||
ends_at DateTime
|
||||
|
|
@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob {
|
|||
stopped_by String? // operator who stopped it early; null when it ended on its own
|
||||
|
||||
@@index([group_id])
|
||||
@@index([api_key_id])
|
||||
@@index([target_type, target_id])
|
||||
@@index([created_at])
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,11 +58,12 @@ def _prisma(jobs=(), attempt_counts=(), attempt_costs=()) -> MagicMock:
|
|||
return prisma
|
||||
|
||||
|
||||
def _job_record(job: ActiveShadowEvalJob, api_key_id="key-hash") -> MagicMock:
|
||||
def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash") -> MagicMock:
|
||||
record = MagicMock()
|
||||
for field, value in dict(
|
||||
id=job.id,
|
||||
api_key_id=api_key_id,
|
||||
target_type=target_type,
|
||||
target_id=target_id,
|
||||
router_name=job.router_name,
|
||||
direction=job.direction,
|
||||
baseline_model=job.baseline_model,
|
||||
|
|
@ -123,7 +124,7 @@ def _spend_counter(store=None):
|
|||
return counter, read, write
|
||||
|
||||
|
||||
def _logger(router=None, prisma=None, jobs=(), counter_store=None) -> ShadowEvalLogger:
|
||||
def _logger(router=None, prisma=None, jobs=(), counter_store=None, jobs_by_target=None) -> ShadowEvalLogger:
|
||||
cache = InMemoryCache(max_size_in_memory=4, default_ttl=60)
|
||||
counter, read, write = _spend_counter(counter_store)
|
||||
funnel_events = []
|
||||
|
|
@ -137,8 +138,9 @@ def _logger(router=None, prisma=None, jobs=(), counter_store=None) -> ShadowEval
|
|||
)
|
||||
logger._test_counter = counter
|
||||
logger._test_funnel = funnel_events
|
||||
if jobs:
|
||||
cache.set_cache("shadow_eval:active_jobs", {"key-hash": tuple(jobs)})
|
||||
seeded = jobs_by_target if jobs_by_target is not None else ({("key", "key-hash"): tuple(jobs)} if jobs else None)
|
||||
if seeded is not None:
|
||||
cache.set_cache("shadow_eval:active_jobs", seeded)
|
||||
return logger
|
||||
|
||||
|
||||
|
|
@ -837,6 +839,86 @@ class TestSuccessHookSkipChain:
|
|||
prisma.db.litellm_shadowevalattempt.create.assert_not_called()
|
||||
|
||||
|
||||
JWT_IDENTITY = {"user_api_key_hash": None, "user_api_key_team_id": "team-eng", "user_api_key_user_id": "dev-alice"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestTargetMatching:
|
||||
"""A request qualifies for a job through ANY of its resolved identities: key hash,
|
||||
team id, or user id. Team and user jobs must therefore sample JWT-authenticated
|
||||
traffic, which carries no key hash at all."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target,sampled",
|
||||
[
|
||||
(("team", "team-eng"), True),
|
||||
(("user", "dev-alice"), True),
|
||||
(("key", "some-key"), False),
|
||||
],
|
||||
ids=["team-job-samples-jwt-traffic", "user-job-samples-jwt-traffic", "key-jobs-never-match-keyless-traffic"],
|
||||
)
|
||||
async def test_jwt_shaped_traffic_matches_team_and_user_jobs_but_no_key_job(self, target, sampled):
|
||||
prisma = _prisma()
|
||||
router = _router()
|
||||
logger = _logger(router=router, prisma=prisma, jobs_by_target={target: (_job(),)})
|
||||
hook_kwargs = _success_kwargs()
|
||||
hook_kwargs["standard_logging_object"]["metadata"] = dict(JWT_IDENTITY)
|
||||
|
||||
await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None)
|
||||
await _drain(logger)
|
||||
|
||||
if sampled:
|
||||
prisma.db.litellm_shadowevalattempt.create.assert_awaited_once()
|
||||
assert prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]["job_id"] == "job-1"
|
||||
else:
|
||||
router.acompletion.assert_not_called()
|
||||
prisma.db.litellm_shadowevalattempt.create.assert_not_called()
|
||||
|
||||
async def test_an_event_with_no_identity_early_returns_without_a_cache_read(self):
|
||||
prisma = _prisma()
|
||||
router = _router()
|
||||
cache = MagicMock(spec=InMemoryCache)
|
||||
cache.async_get_cache = AsyncMock()
|
||||
logger = ShadowEvalLogger(
|
||||
router_provider=lambda: router,
|
||||
prisma_provider=lambda: prisma,
|
||||
jobs_cache=cache,
|
||||
)
|
||||
hook_kwargs = _success_kwargs()
|
||||
hook_kwargs["standard_logging_object"]["metadata"] = {}
|
||||
|
||||
await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None)
|
||||
|
||||
cache.async_get_cache.assert_not_awaited()
|
||||
router.acompletion.assert_not_called()
|
||||
prisma.db.litellm_shadowevalattempt.create.assert_not_called()
|
||||
|
||||
async def test_an_event_matching_a_key_job_and_a_team_job_fires_both(self):
|
||||
"""A request's key and its team can each hold a job; the two are separately
|
||||
budgeted experiments, so both fire and each counts its own start."""
|
||||
prisma = _prisma()
|
||||
logger = _logger(
|
||||
router=_router(),
|
||||
prisma=prisma,
|
||||
jobs_by_target={
|
||||
("key", "key-hash"): (_job(id="key-job"),),
|
||||
("team", "team-eng"): (_job(id="team-job"),),
|
||||
},
|
||||
)
|
||||
hook_kwargs = _success_kwargs()
|
||||
hook_kwargs["standard_logging_object"]["metadata"] = {
|
||||
"user_api_key_hash": "key-hash",
|
||||
"user_api_key_team_id": "team-eng",
|
||||
}
|
||||
|
||||
await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None)
|
||||
await _drain(logger)
|
||||
|
||||
rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.call_args_list]
|
||||
assert sorted(row["job_id"] for row in rows) == ["key-job", "team-job"]
|
||||
assert logger._job_starts == {"key-job": 1, "team-job": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestActiveJobsCache:
|
||||
async def test_cache_miss_reads_db_once_then_serves_from_cache(self):
|
||||
|
|
@ -851,8 +933,8 @@ class TestActiveJobsCache:
|
|||
first = await logger._active_jobs()
|
||||
second = await logger._active_jobs()
|
||||
|
||||
assert [job.id for job in first["key-hash"]] == ["job-1"]
|
||||
assert second["key-hash"][0].attempts == 7
|
||||
assert [job.id for job in first[("key", "key-hash")]] == ["job-1"]
|
||||
assert second[("key", "key-hash")][0].attempts == 7
|
||||
assert prisma.db.litellm_shadowevaljob.find_many.await_count == 1
|
||||
where = prisma.db.litellm_shadowevaljob.find_many.call_args.kwargs["where"]
|
||||
assert where["stopped_at"] is None
|
||||
|
|
@ -899,8 +981,8 @@ class TestActiveJobsCache:
|
|||
jobs = await logger._active_jobs()
|
||||
|
||||
assert logger._job_starts == {}
|
||||
assert jobs["key-hash"][0].attempts == 7
|
||||
assert jobs["key-hash"][0].spend == 0.05
|
||||
assert jobs[("key", "key-hash")][0].attempts == 7
|
||||
assert jobs[("key", "key-hash")][0].spend == 0.05
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1249,13 +1331,14 @@ class TestActiveJobsFailClosed:
|
|||
jobs_cache=InMemoryCache(max_size_in_memory=4, default_ttl=60),
|
||||
)
|
||||
|
||||
assert [job.id for job in (await logger._active_jobs())["key-hash"]] == ["job-ok"]
|
||||
assert [job.id for job in (await logger._active_jobs())[("key", "key-hash")]] == ["job-ok"]
|
||||
|
||||
async def test_both_of_a_key_s_jobs_survive_the_lookup(self):
|
||||
async def test_every_targets_jobs_survive_the_lookup_keyed_by_type_and_id(self):
|
||||
records = [
|
||||
_job_record(_job(id="job-forward")),
|
||||
_job_record(_reverse_job(id="job-reverse")),
|
||||
_job_record(_job(id="job-other"), api_key_id="other-key"),
|
||||
_job_record(_job(id="job-other"), target_id="other-key"),
|
||||
_job_record(_job(id="job-team"), target_type="team", target_id="team-eng"),
|
||||
]
|
||||
prisma = _prisma(jobs=records, attempt_counts=[("job-reverse", 3)])
|
||||
logger = ShadowEvalLogger(
|
||||
|
|
@ -1266,9 +1349,11 @@ class TestActiveJobsFailClosed:
|
|||
|
||||
jobs = await logger._active_jobs()
|
||||
|
||||
assert sorted(job.id for job in jobs["key-hash"]) == ["job-forward", "job-reverse"]
|
||||
assert [job.id for job in jobs["other-key"]] == ["job-other"]
|
||||
assert {job.id: job.attempts for job in jobs["key-hash"]}["job-reverse"] == 3
|
||||
assert sorted(job.id for job in jobs[("key", "key-hash")]) == ["job-forward", "job-reverse"]
|
||||
assert [job.id for job in jobs[("key", "other-key")]] == ["job-other"]
|
||||
assert [job.id for job in jobs[("team", "team-eng")]] == ["job-team"]
|
||||
assert ("team-eng",) not in jobs and "team-eng" not in jobs
|
||||
assert {job.id: job.attempts for job in jobs[("key", "key-hash")]}["job-reverse"] == 3
|
||||
|
||||
|
||||
def _failing_router():
|
||||
|
|
|
|||
|
|
@ -878,7 +878,8 @@ def _leg_record(**overrides: object) -> MagicMock:
|
|||
defaults = {
|
||||
"id": "leg-1",
|
||||
"group_id": "job-1",
|
||||
"api_key_id": "key-hash",
|
||||
"target_type": "key",
|
||||
"target_id": "key-hash",
|
||||
"router_name": "my-router",
|
||||
"direction": "forward",
|
||||
"baseline_model": None,
|
||||
|
|
@ -912,8 +913,28 @@ def _key_record(
|
|||
return record
|
||||
|
||||
|
||||
def _team_record(team_id: str, team_alias: str | None) -> MagicMock:
|
||||
record = MagicMock(spec=["team_id", "team_alias"])
|
||||
record.team_id = team_id
|
||||
record.team_alias = team_alias
|
||||
return record
|
||||
|
||||
|
||||
def _user_record(user_id: str, user_email: str | None) -> MagicMock:
|
||||
record = MagicMock(spec=["user_id", "user_email"])
|
||||
record.user_id = user_id
|
||||
record.user_email = user_email
|
||||
return record
|
||||
|
||||
|
||||
def _shadow_prisma(
|
||||
legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-hash", "key-hash-2"), key_teams=None
|
||||
legs=(),
|
||||
agg_rows=None,
|
||||
by_leg_rows=None,
|
||||
known_keys=("key-hash", "key-hash-2"),
|
||||
key_teams=None,
|
||||
known_teams=None,
|
||||
known_users=None,
|
||||
) -> MagicMock:
|
||||
"""The job-table fake honours the filters it is handed, so a read that forgets
|
||||
stopped_at sees rows the partial index would have released, one that forgets
|
||||
|
|
@ -921,6 +942,8 @@ def _shadow_prisma(
|
|||
group read that matched on a leg id would come back empty."""
|
||||
prisma = MagicMock()
|
||||
teams: Final = key_teams or {}
|
||||
team_aliases: Final = known_teams or {}
|
||||
user_emails: Final = known_users or {}
|
||||
|
||||
async def find_tokens(*, where):
|
||||
"""Honours the token filter, like the job-table fake below: the endpoint derives the
|
||||
|
|
@ -931,6 +954,17 @@ def _shadow_prisma(
|
|||
|
||||
prisma.db.litellm_verificationtoken.find_many = AsyncMock(side_effect=find_tokens)
|
||||
|
||||
async def find_teams(*, where):
|
||||
requested = where["team_id"]["in"]
|
||||
return [_team_record(t, alias) for t, alias in team_aliases.items() if t in requested]
|
||||
|
||||
async def find_users(*, where):
|
||||
requested = where["user_id"]["in"]
|
||||
return [_user_record(u, email) for u, email in user_emails.items() if u in requested]
|
||||
|
||||
prisma.db.litellm_teamtable.find_many = AsyncMock(side_effect=find_teams)
|
||||
prisma.db.litellm_usertable.find_many = AsyncMock(side_effect=find_users)
|
||||
|
||||
async def execute_raw(sql: str, *params: object):
|
||||
if "SET stopped_by" in sql:
|
||||
group = [row for row in stored if row.group_id == params[0]]
|
||||
|
|
@ -959,9 +993,19 @@ def _shadow_prisma(
|
|||
async def find_many_legs(where=None, **_: object):
|
||||
current = list(stored)
|
||||
w = dict(where or {})
|
||||
if "api_key_id" in w:
|
||||
wanted = w["api_key_id"]["in"] if isinstance(w["api_key_id"], dict) else [w["api_key_id"]]
|
||||
current = [row for row in current if row.api_key_id in wanted]
|
||||
if "OR" in w:
|
||||
pairs = [
|
||||
(
|
||||
branch["target_type"],
|
||||
branch["target_id"]["in"] if isinstance(branch["target_id"], dict) else [branch["target_id"]],
|
||||
)
|
||||
for branch in w["OR"]
|
||||
]
|
||||
current = [
|
||||
row
|
||||
for row in current
|
||||
if any(row.target_type == target_type and row.target_id in ids for target_type, ids in pairs)
|
||||
]
|
||||
if "direction" in w:
|
||||
current = [row for row in current if row.direction == w["direction"]]
|
||||
if "stopped_at" in w:
|
||||
|
|
@ -983,7 +1027,8 @@ def _shadow_prisma(
|
|||
fields = (
|
||||
"id",
|
||||
"group_id",
|
||||
"api_key_id",
|
||||
"target_type",
|
||||
"target_id",
|
||||
"router_name",
|
||||
"direction",
|
||||
"baseline_model",
|
||||
|
|
@ -1009,7 +1054,11 @@ def _shadow_prisma(
|
|||
if "AS attempt_count" in sql:
|
||||
return prisma.attempt_rows
|
||||
if "GROUP BY group_id" in sql:
|
||||
scoped = [row for row in stored if "api_key_id = $2" not in sql or row.api_key_id == params[1]]
|
||||
scoped = [
|
||||
row
|
||||
for row in stored
|
||||
if "target_type = $2" not in sql or (row.target_type == params[1] and row.target_id == params[2])
|
||||
]
|
||||
keep = set(newest_groups(scoped, params[0]))
|
||||
return [leg_dict(row) for row in stored if row.group_id in keep]
|
||||
if "FILTER (WHERE outcome != 'error')::int AS judged_count" in sql:
|
||||
|
|
@ -1058,7 +1107,7 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp
|
|||
|
||||
response = await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN)
|
||||
|
||||
sweep_sql, sweep_keys = prisma.db.execute_raw.call_args.args
|
||||
sweep_sql, sweep_ids, sweep_type = prisma.db.execute_raw.call_args.args
|
||||
assert "stopped_at IS NULL" in sweep_sql
|
||||
assert "j.ends_at <= (NOW() AT TIME ZONE 'utc')" in sweep_sql
|
||||
assert "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in sweep_sql
|
||||
|
|
@ -1066,12 +1115,13 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp
|
|||
assert "j.max_budget IS NOT NULL" in sweep_sql
|
||||
assert ">= j.max_budget" in sweep_sql
|
||||
assert "SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost)" in sweep_sql
|
||||
assert "j.api_key_id = ANY($1::text[])" in sweep_sql
|
||||
assert sweep_keys == ["key-hash", "key-hash-2"]
|
||||
assert "j.target_type = $2 AND j.target_id = ANY($1::text[])" in sweep_sql
|
||||
assert sweep_ids == ["key-hash", "key-hash-2"]
|
||||
assert sweep_type == "key"
|
||||
prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once()
|
||||
rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
|
||||
assert [row["api_key_id"] for row in rows] == ["key-hash", "key-hash-2"]
|
||||
assert len({frozenset((k, v) for k, v in row.items() if k not in ("api_key_id", "id")) for row in rows}) == 1
|
||||
assert [(row["target_type"], row["target_id"]) for row in rows] == [("key", "key-hash"), ("key", "key-hash-2")]
|
||||
assert len({frozenset((k, v) for k, v in row.items() if k not in ("target_id", "id")) for row in rows}) == 1
|
||||
assert len({row["id"] for row in rows}) == len(rows)
|
||||
assert len({row["group_id"] for row in rows}) == 1
|
||||
assert all(row["max_turns"] == SHADOW_EVAL_TURN_VALVE and row["created_by"] == "admin" for row in rows)
|
||||
|
|
@ -1080,11 +1130,12 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp
|
|||
assert response.job_id == rows[0]["group_id"]
|
||||
assert response.status == "running"
|
||||
assert response.judged_count is None
|
||||
assert [(key.api_key_id, key.max_budget, key.key_alias) for key in response.keys] == [
|
||||
assert [(target.target_id, target.max_budget, target.target_alias) for target in response.targets] == [
|
||||
("key-hash", 5.0, "prod-alpha"),
|
||||
("key-hash-2", 5.0, "prod-alpha"),
|
||||
]
|
||||
assert all(key.max_turns == SHADOW_EVAL_TURN_VALVE for key in response.keys)
|
||||
assert all(target.target_type == "key" for target in response.targets)
|
||||
assert all(target.max_turns == SHADOW_EVAL_TURN_VALVE for target in response.targets)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1232,7 +1283,7 @@ async def test_start_shadow_eval_rejections(
|
|||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
_configure_anthropic_sdk_judge(monkeypatch)
|
||||
prisma = _shadow_prisma(legs=[_leg_record(id=f"leg-{key}", group_id="job-7", api_key_id=key) for key in claimed])
|
||||
prisma = _shadow_prisma(legs=[_leg_record(id=f"leg-{key}", group_id="job-7", target_id=key) for key in claimed])
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
|
||||
|
|
@ -1315,14 +1366,14 @@ async def test_start_shadow_eval_names_the_busy_key_and_its_job(monkeypatch: pyt
|
|||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
_configure_anthropic_sdk_judge(monkeypatch)
|
||||
prisma = _shadow_prisma(legs=[_leg_record(id="leg-b", group_id="job-7", api_key_id="key-hash-2")])
|
||||
prisma = _shadow_prisma(legs=[_leg_record(id="leg-b", group_id="job-7", target_id="key-hash-2")])
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN)
|
||||
assert exc.value.status_code == 409
|
||||
assert "key-hash-2 (job job-7)" in exc.value.detail
|
||||
assert "key key-hash-2 (job job-7)" in exc.value.detail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1441,6 +1492,180 @@ def test_start_shadow_eval_request_dedupes_and_bounds_the_key_set():
|
|||
_start_request(api_key_ids=tuple(f"k{i}" for i in range(101)))
|
||||
|
||||
|
||||
def test_start_request_bounds_the_combined_target_count_across_types():
|
||||
"""The 1..100 bound counts keys, teams, and users together, so a caller cannot dodge
|
||||
it by spreading targets over the three fields, and a request naming no target of any
|
||||
type samples nothing and is rejected."""
|
||||
with pytest.raises(ValidationError, match="at least one target"):
|
||||
_start_request(api_key_ids=(), team_ids=(), user_ids=())
|
||||
with pytest.raises(ValidationError, match="at most 100 targets"):
|
||||
_start_request(api_key_ids=tuple(f"k{i}" for i in range(60)), team_ids=tuple(f"t{i}" for i in range(41)))
|
||||
mixed = _start_request(api_key_ids=tuple(f"k{i}" for i in range(60)), team_ids=tuple(f"t{i}" for i in range(40)))
|
||||
assert len(mixed.api_key_ids) + len(mixed.team_ids) == 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"overrides,prisma_kwargs,expected_target",
|
||||
[
|
||||
(
|
||||
{"api_key_ids": (), "team_ids": ("team-eng",)},
|
||||
{"known_teams": {"team-eng": "Engineering"}},
|
||||
("team", "team-eng", "Engineering"),
|
||||
),
|
||||
(
|
||||
{"api_key_ids": (), "user_ids": ("dev-alice",)},
|
||||
{"known_users": {"dev-alice": "alice@example.com"}},
|
||||
("user", "dev-alice", "alice@example.com"),
|
||||
),
|
||||
],
|
||||
ids=["team-target-labeled-by-team-alias", "user-target-labeled-by-user-email"],
|
||||
)
|
||||
async def test_start_shadow_eval_creates_typed_legs_for_team_and_user_targets(
|
||||
monkeypatch: pytest.MonkeyPatch, overrides, prisma_kwargs, expected_target
|
||||
):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
_configure_anthropic_sdk_judge(monkeypatch)
|
||||
prisma = _shadow_prisma(**prisma_kwargs)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
|
||||
response = await start_shadow_eval(_start_request(**overrides), ADMIN)
|
||||
|
||||
target_type, target_id, target_alias = expected_target
|
||||
rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
|
||||
assert [(row["target_type"], row["target_id"]) for row in rows] == [(target_type, target_id)]
|
||||
assert response.status == "running"
|
||||
target = response.targets[0]
|
||||
assert (target.target_type, target.target_id, target.target_alias, target.key_name) == (
|
||||
target_type,
|
||||
target_id,
|
||||
target_alias,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"overrides,prisma_kwargs,expected_detail",
|
||||
[
|
||||
(
|
||||
{"api_key_ids": (), "team_ids": ("team-eng", "team-ghost")},
|
||||
{"known_teams": {"team-eng": "Engineering"}},
|
||||
"team_ids not on this proxy: team-ghost",
|
||||
),
|
||||
(
|
||||
{"api_key_ids": (), "user_ids": ("dev-alice", "dev-ghost")},
|
||||
{"known_users": {"dev-alice": "alice@example.com"}},
|
||||
"user_ids not on this proxy: dev-ghost",
|
||||
),
|
||||
],
|
||||
ids=["unknown-team", "unknown-user"],
|
||||
)
|
||||
async def test_start_shadow_eval_rejects_teams_and_users_this_proxy_does_not_know(
|
||||
monkeypatch: pytest.MonkeyPatch, overrides, prisma_kwargs, expected_detail
|
||||
):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
_configure_anthropic_sdk_judge(monkeypatch)
|
||||
prisma = _shadow_prisma(**prisma_kwargs)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await start_shadow_eval(_start_request(**overrides), ADMIN)
|
||||
assert exc.value.status_code == 400
|
||||
assert expected_detail in exc.value.detail
|
||||
prisma.db.litellm_shadowevaljob.create_many.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_shadow_eval_mixed_targets_create_both_legs_and_sweep_once_per_type(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
_configure_anthropic_sdk_judge(monkeypatch)
|
||||
prisma = _shadow_prisma(known_teams={"team-eng": "Engineering"})
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
|
||||
response = await start_shadow_eval(_start_request(team_ids=("team-eng",)), ADMIN)
|
||||
|
||||
rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"]
|
||||
assert [(row["target_type"], row["target_id"]) for row in rows] == [("key", "key-hash"), ("team", "team-eng")]
|
||||
assert len({row["group_id"] for row in rows}) == 1
|
||||
sweeps = [
|
||||
call.args
|
||||
for call in prisma.db.execute_raw.await_args_list
|
||||
if "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in call.args[0]
|
||||
]
|
||||
assert [(ids, target_type) for _, ids, target_type in sweeps] == [(["key-hash"], "key"), (["team-eng"], "team")]
|
||||
assert [(t.target_type, t.target_id, t.target_alias) for t in response.targets] == [
|
||||
("key", "key-hash", "prod-alpha"),
|
||||
("team", "team-eng", "Engineering"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_shadow_eval_names_the_busy_team_target(monkeypatch: pytest.MonkeyPatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
_configure_anthropic_sdk_judge(monkeypatch)
|
||||
prisma = _shadow_prisma(
|
||||
legs=[_leg_record(id="leg-t", group_id="job-7", target_type="team", target_id="team-eng")],
|
||||
known_teams={"team-eng": "Engineering"},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await start_shadow_eval(_start_request(api_key_ids=(), team_ids=("team-eng",)), ADMIN)
|
||||
assert exc.value.status_code == 409
|
||||
assert "team team-eng (job job-7)" in exc.value.detail
|
||||
prisma.db.litellm_shadowevaljob.create_many.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_shadow_eval_claim_matches_exact_target_pairs_not_bare_ids(monkeypatch: pytest.MonkeyPatch):
|
||||
"""A key whose hash happens to spell a team's id must not hold the team's slot: the
|
||||
claim matches (target_type, target_id) pairs, never ids across kinds."""
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
_configure_anthropic_sdk_judge(monkeypatch)
|
||||
prisma = _shadow_prisma(
|
||||
legs=[_leg_record(id="leg-k", group_id="job-7", target_type="key", target_id="team-eng")],
|
||||
known_teams={"team-eng": "Engineering"},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
|
||||
response = await start_shadow_eval(_start_request(api_key_ids=(), team_ids=("team-eng",)), ADMIN)
|
||||
|
||||
assert response.status == "running"
|
||||
prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_shadow_eval_jobs_rejects_a_lone_filter_half(monkeypatch: pytest.MonkeyPatch):
|
||||
"""target_type and target_id only mean anything together: a bare id could name a key
|
||||
or a team, and a bare type filters nothing."""
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
prisma = _shadow_prisma(legs=[_leg_record()])
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
with pytest.raises(HTTPException) as id_only:
|
||||
await list_shadow_eval_jobs(VIEWER, target_type=None, target_id="key-hash", limit=50)
|
||||
assert id_only.value.status_code == 400
|
||||
|
||||
with pytest.raises(HTTPException) as type_only:
|
||||
await list_shadow_eval_jobs(VIEWER, target_type="key", target_id=None, limit=50)
|
||||
assert type_only.value.status_code == 400
|
||||
prisma.db.query_raw.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
|
@ -1530,7 +1755,7 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke
|
|||
},
|
||||
]
|
||||
prisma = _shadow_prisma(
|
||||
legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=50)],
|
||||
legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2", max_turns=50)],
|
||||
agg_rows=tier_rows,
|
||||
by_leg_rows=leg_rows,
|
||||
)
|
||||
|
|
@ -1549,8 +1774,10 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke
|
|||
assert response.results.by_tier[0].shadow_win_rate_pct == 50.0
|
||||
assert response.results.overall_shadow_win_rate_pct == 40.0
|
||||
assert response.results.overall_tie_rate_pct == 20.0
|
||||
assert [(s.group, s.turn_count) for s in response.results.by_key] == [("key-hash", 6), ("key-hash-2", 4)]
|
||||
assert response.results.by_key[0].shadow_win_rate_pct == 66.7
|
||||
verdicts_by_target = {(t.target_type, t.target_id): t.verdicts for t in response.targets}
|
||||
assert verdicts_by_target[("key", "key-hash")].turn_count == 6
|
||||
assert verdicts_by_target[("key", "key-hash")].shadow_win_rate_pct == 66.7
|
||||
assert verdicts_by_target[("key", "key-hash-2")].turn_count == 4
|
||||
agg_sql = next(call.args[0] for call in prisma.db.query_raw.await_args_list if "real_spend" in call.args[0])
|
||||
assert agg_sql.count("FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit)") == 2
|
||||
assert response.results.by_tier[0].real_spend == 0.08
|
||||
|
|
@ -1561,7 +1788,10 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke
|
|||
assert response.results.not_sampled_count is None
|
||||
assert response.results.unjudgeable_count is None
|
||||
assert response.results.shed_count is None
|
||||
assert [(key.api_key_id, key.max_turns) for key in response.keys] == [("key-hash", 200), ("key-hash-2", 50)]
|
||||
assert [(target.target_id, target.max_turns) for target in response.targets] == [
|
||||
("key-hash", 200),
|
||||
("key-hash-2", 50),
|
||||
]
|
||||
totals_args = [call.args for call in prisma.db.query_raw.await_args_list if "judged_count" in call.args[0]]
|
||||
assert totals_args == [(totals_args[0][0], ["leg-1", "leg-2"])]
|
||||
error_where = prisma.db.litellm_shadowevalattempt.find_first.call_args.kwargs["where"]
|
||||
|
|
@ -1595,7 +1825,7 @@ async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monke
|
|||
_leg_record(created_at=datetime(2026, 8, 13, tzinfo=timezone.utc)),
|
||||
_leg_record(
|
||||
id="leg-2",
|
||||
api_key_id="key-hash-2",
|
||||
target_id="key-hash-2",
|
||||
stopped_at=stamp,
|
||||
created_at=datetime(2026, 8, 13, tzinfo=timezone.utc),
|
||||
),
|
||||
|
|
@ -1615,14 +1845,14 @@ async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monke
|
|||
)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50)
|
||||
|
||||
assert [(job.job_id, job.status) for job in jobs] == [
|
||||
("job-1", "running"),
|
||||
("job-2", "stopped"),
|
||||
("job-3", "completed"),
|
||||
]
|
||||
assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"]
|
||||
assert [target.target_id for target in jobs[0].targets] == ["key-hash", "key-hash-2"]
|
||||
assert all(job.judged_count is None and job.results is None for job in jobs)
|
||||
legs_sql, legs_limit = prisma.db.query_raw.await_args_list[0].args
|
||||
assert "GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int" in legs_sql
|
||||
|
|
@ -1648,17 +1878,20 @@ async def test_list_shadow_eval_jobs_filters_to_jobs_containing_the_key(monkeypa
|
|||
prisma = _shadow_prisma(
|
||||
legs=[
|
||||
_leg_record(),
|
||||
_leg_record(id="leg-2", api_key_id="key-hash-2"),
|
||||
_leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash-2"),
|
||||
_leg_record(id="leg-2", target_id="key-hash-2"),
|
||||
_leg_record(id="leg-3", group_id="job-2", target_id="key-hash-2"),
|
||||
_leg_record(id="leg-4", group_id="job-3"),
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id="key-hash-2", limit=50)
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, target_type="key", target_id="key-hash-2", limit=50)
|
||||
|
||||
assert [job.job_id for job in jobs] == ["job-1", "job-2"]
|
||||
assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"]
|
||||
assert [target.target_id for target in jobs[0].targets] == ["key-hash", "key-hash-2"]
|
||||
legs_sql, *legs_params = prisma.db.query_raw.await_args_list[0].args
|
||||
assert "WHERE target_type = $2 AND target_id = $3" in legs_sql
|
||||
assert legs_params == [50, "key", "key-hash-2"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -1682,7 +1915,7 @@ async def test_job_status_runs_until_every_key_stops_and_completed_outranks_stop
|
|||
legs=[
|
||||
_leg_record(
|
||||
id=f"leg-{index}",
|
||||
api_key_id=f"key-{index}",
|
||||
target_id=f"key-{index}",
|
||||
stopped_at=stamp if stopped else None,
|
||||
ends_at=datetime.now(timezone.utc) + timedelta(days=days_left),
|
||||
)
|
||||
|
|
@ -1691,7 +1924,7 @@ async def test_job_status_runs_until_every_key_stops_and_completed_outranks_stop
|
|||
)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50)
|
||||
|
||||
assert [job.status for job in jobs] == [expected]
|
||||
|
||||
|
|
@ -1707,9 +1940,9 @@ async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch
|
|||
prisma = _shadow_prisma(
|
||||
legs=[
|
||||
_leg_record(max_turns=5),
|
||||
_leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=5),
|
||||
_leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=5),
|
||||
_leg_record(id="leg-4", group_id="job-2", api_key_id="key-hash-2", max_turns=5),
|
||||
_leg_record(id="leg-2", target_id="key-hash-2", max_turns=5),
|
||||
_leg_record(id="leg-3", group_id="job-2", target_id="key-hash", max_turns=5),
|
||||
_leg_record(id="leg-4", group_id="job-2", target_id="key-hash-2", max_turns=5),
|
||||
]
|
||||
)
|
||||
prisma.attempt_rows = [
|
||||
|
|
@ -1720,13 +1953,13 @@ async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch
|
|||
]
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50)
|
||||
|
||||
by_id = {job.job_id: job for job in jobs}
|
||||
assert by_id["job-1"].status == "completed"
|
||||
assert all(key.stopped_at is None for key in by_id["job-1"].keys)
|
||||
assert all(target.stopped_at is None for target in by_id["job-1"].targets)
|
||||
assert by_id["job-2"].status == "running"
|
||||
assert {key.api_key_id: key.attempt_count for key in by_id["job-2"].keys} == {"key-hash": 5, "key-hash-2": 3}
|
||||
assert {t.target_id: t.attempt_count for t in by_id["job-2"].targets} == {"key-hash": 5, "key-hash-2": 3}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1740,7 +1973,7 @@ async def test_recorded_operator_stop_outranks_budget_arithmetic(monkeypatch: py
|
|||
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6, "spend": 0.0}]
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50)
|
||||
assert jobs[0].status == "stopped"
|
||||
assert jobs[0].stopped_by == "admin"
|
||||
|
||||
|
|
@ -1760,7 +1993,7 @@ async def test_backfilled_legacy_stop_never_reads_as_completion(monkeypatch: pyt
|
|||
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6, "spend": 0.0}]
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50)
|
||||
assert jobs[0].status == "stopped"
|
||||
|
||||
|
||||
|
|
@ -1808,6 +2041,56 @@ def test_max_budget_migration_is_additive_and_leaves_legacy_rows_null():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.asyncio
|
||||
async def test_verdicts_keep_same_id_targets_of_different_kinds_distinct(monkeypatch):
|
||||
"""A team and a user can legitimately share an id; their slices must not merge."""
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
leg_rows = [
|
||||
{
|
||||
"grp": "leg-1",
|
||||
"turn_count": 6,
|
||||
"real_wins": 2,
|
||||
"shadow_wins": 4,
|
||||
"ties": 0,
|
||||
"avg_confidence": 0.8,
|
||||
"real_spend": 0.02,
|
||||
"shadow_spend": 0.01,
|
||||
"cache_hit_turns": 0,
|
||||
},
|
||||
{
|
||||
"grp": "leg-2",
|
||||
"turn_count": 4,
|
||||
"real_wins": 3,
|
||||
"shadow_wins": 0,
|
||||
"ties": 1,
|
||||
"avg_confidence": 0.6,
|
||||
"real_spend": 0.05,
|
||||
"shadow_spend": 0.04,
|
||||
"cache_hit_turns": 1,
|
||||
},
|
||||
]
|
||||
prisma = _shadow_prisma(
|
||||
legs=[
|
||||
_leg_record(target_type="team", target_id="dev-alice"),
|
||||
_leg_record(id="leg-2", target_type="user", target_id="dev-alice"),
|
||||
],
|
||||
agg_rows=leg_rows[:1],
|
||||
by_leg_rows=leg_rows,
|
||||
known_teams={"dev-alice": "alias"},
|
||||
known_users={"dev-alice": "alice@example.com"},
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
response = await get_shadow_eval_job("job-1", VIEWER)
|
||||
|
||||
verdicts_by_target = {(t.target_type, t.target_id): t.verdicts for t in response.targets}
|
||||
assert verdicts_by_target[("team", "dev-alice")].turn_count == 6
|
||||
assert verdicts_by_target[("team", "dev-alice")].shadow_win_rate_pct == 66.7
|
||||
assert verdicts_by_target[("user", "dev-alice")].turn_count == 4
|
||||
assert verdicts_by_target[("user", "dev-alice")].shadow_win_rate_pct == 0.0
|
||||
|
||||
|
||||
async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pytest.MonkeyPatch):
|
||||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
|
|
@ -1832,9 +2115,9 @@ async def test_list_reads_completed_once_every_key_spends_its_dollar_budget(monk
|
|||
prisma = _shadow_prisma(
|
||||
legs=[
|
||||
_leg_record(max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0),
|
||||
_leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0),
|
||||
_leg_record(id="leg-2", target_id="key-hash-2", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0),
|
||||
_leg_record(
|
||||
id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0
|
||||
id="leg-3", group_id="job-2", target_id="key-hash", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0
|
||||
),
|
||||
]
|
||||
)
|
||||
|
|
@ -1845,13 +2128,13 @@ async def test_list_reads_completed_once_every_key_spends_its_dollar_budget(monk
|
|||
]
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50)
|
||||
|
||||
by_id = {job.job_id: job for job in jobs}
|
||||
assert by_id["job-1"].status == "completed"
|
||||
assert by_id["job-2"].status == "running"
|
||||
assert {key.api_key_id: key.spend for key in by_id["job-1"].keys} == {"key-hash": 1.0, "key-hash-2": 1.25}
|
||||
assert all(key.max_budget == 1.0 for key in by_id["job-1"].keys)
|
||||
assert {t.target_id: t.spend for t in by_id["job-1"].targets} == {"key-hash": 1.0, "key-hash-2": 1.25}
|
||||
assert all(target.max_budget == 1.0 for target in by_id["job-1"].targets)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1880,11 +2163,11 @@ async def test_legacy_jobs_without_a_dollar_budget_stay_turn_gated(monkeypatch:
|
|||
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 40, "spend": 250.0}]
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50)
|
||||
|
||||
assert jobs[0].status == "running"
|
||||
assert jobs[0].keys[0].max_budget is None
|
||||
assert jobs[0].keys[0].spend == 250.0
|
||||
assert jobs[0].targets[0].max_budget is None
|
||||
assert jobs[0].targets[0].spend == 250.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1892,14 +2175,14 @@ async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest
|
|||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
prisma = _shadow_prisma(
|
||||
legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="deleted-key-hash")],
|
||||
legs=[_leg_record(), _leg_record(id="leg-2", target_id="deleted-key-hash")],
|
||||
known_keys=("key-hash", "key-hash-2"),
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", _shadow_router())
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
|
||||
assert [(key.key_alias, key.key_name) for key in jobs[0].keys] == [
|
||||
jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50)
|
||||
assert [(target.target_alias, target.key_name) for target in jobs[0].targets] == [
|
||||
(None, None),
|
||||
("prod-alpha", "sk-...lpha"),
|
||||
]
|
||||
|
|
@ -1907,7 +2190,7 @@ async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest
|
|||
assert batched_where == {"token": {"in": ["deleted-key-hash", "key-hash"]}}
|
||||
|
||||
detail = await get_shadow_eval_job("job-1", VIEWER)
|
||||
assert [key.key_alias for key in detail.keys] == [None, "prod-alpha"]
|
||||
assert [target.target_alias for target in detail.targets] == [None, "prod-alpha"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1919,7 +2202,7 @@ async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_runnin
|
|||
import litellm.proxy.proxy_server as proxy_server
|
||||
|
||||
earned = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||
prisma = _shadow_prisma(legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", stopped_at=earned)])
|
||||
prisma = _shadow_prisma(legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2", stopped_at=earned)])
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
stopped = await stop_shadow_eval_job("job-1", ADMIN)
|
||||
|
|
@ -1938,9 +2221,9 @@ async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_runnin
|
|||
assert datetime.fromisoformat(stop_stamp).tzinfo is None
|
||||
assert prisma.db.execute_raw.await_count == 1
|
||||
prisma.db.litellm_shadowevaljob.update_many.assert_not_called()
|
||||
by_key = {key.api_key_id: key.stopped_at for key in stopped.keys}
|
||||
assert by_key["key-hash-2"] == earned
|
||||
assert by_key["key-hash"] is not None and by_key["key-hash"] != earned
|
||||
by_target = {target.target_id: target.stopped_at for target in stopped.targets}
|
||||
assert by_target["key-hash-2"] == earned
|
||||
assert by_target["key-hash"] is not None and by_target["key-hash"] != earned
|
||||
|
||||
done_leg = _leg_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1))
|
||||
prisma_done = _shadow_prisma(legs=[done_leg])
|
||||
|
|
@ -2331,7 +2614,7 @@ async def test_get_shadow_eval_job_sums_funnel_rows_across_legs(monkeypatch: pyt
|
|||
},
|
||||
]
|
||||
prisma = _shadow_prisma(
|
||||
legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2")],
|
||||
legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2")],
|
||||
agg_rows=tier_rows,
|
||||
)
|
||||
prisma.funnel_rows = [{"legs_with_rows": 2, "not_sampled": 30, "unjudgeable": 5, "shed": 2, "withheld": 3}]
|
||||
|
|
@ -2366,7 +2649,7 @@ async def test_partially_seeded_funnel_reads_as_unknown_coverage(monkeypatch: py
|
|||
},
|
||||
]
|
||||
prisma = _shadow_prisma(
|
||||
legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2")],
|
||||
legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2")],
|
||||
agg_rows=tier_rows,
|
||||
)
|
||||
prisma.funnel_rows = [{"legs_with_rows": 1, "not_sampled": 30, "unjudgeable": 5, "shed": 2, "withheld": 0}]
|
||||
|
|
|
|||
|
|
@ -39,6 +39,35 @@ vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({
|
|||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({
|
||||
useInfiniteTeams: vi.fn(() => ({
|
||||
data: { pages: [{ teams: [{ team_id: "team-eng", team_alias: "engineering" }], page: 1, total_pages: 1 }] },
|
||||
isLoading: false,
|
||||
fetchNextPage: vi.fn(),
|
||||
hasNextPage: false,
|
||||
isFetchingNextPage: false,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/users/useUsers", () => ({
|
||||
useInfiniteUsers: vi.fn(() => ({
|
||||
data: {
|
||||
pages: [
|
||||
{
|
||||
users: [{ user_id: "dev-alice", user_alias: null, user_email: "alice@example.com" }],
|
||||
page: 1,
|
||||
total_pages: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
isPending: false,
|
||||
isError: false,
|
||||
fetchNextPage: vi.fn(),
|
||||
hasNextPage: false,
|
||||
isFetchingNextPage: false,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({
|
||||
useAutoRouters: vi.fn(() => ({
|
||||
data: [
|
||||
|
|
@ -60,7 +89,7 @@ vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({
|
|||
})),
|
||||
}));
|
||||
|
||||
import ShadowEvalSection, { shadowedKeyLabel } from "./ShadowEvalSection";
|
||||
import ShadowEvalSection, { shadowedTargetLabel } from "./ShadowEvalSection";
|
||||
import {
|
||||
useShadowEvalJob,
|
||||
useShadowEvalJobs,
|
||||
|
|
@ -77,14 +106,15 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
|
|||
baseline_model: null,
|
||||
judge_model: "anthropic/claude-sonnet-5",
|
||||
shadow_percentage: 10,
|
||||
keys: [
|
||||
targets: [
|
||||
{
|
||||
api_key_id: "hashed-key-abc",
|
||||
target_type: "key",
|
||||
target_id: "hashed-key-abc",
|
||||
max_turns: 10000,
|
||||
max_budget: 10,
|
||||
spend: 3.21,
|
||||
stopped_at: null,
|
||||
key_alias: "prod-alpha",
|
||||
target_alias: "prod-alpha",
|
||||
key_name: "sk-...alpha",
|
||||
},
|
||||
],
|
||||
|
|
@ -129,7 +159,6 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
|
|||
cache_hit_turns: 2,
|
||||
},
|
||||
],
|
||||
by_key: [],
|
||||
overall_shadow_win_rate_pct: 48.0,
|
||||
overall_tie_rate_pct: 22.0,
|
||||
sampled_real_spend: 0.6,
|
||||
|
|
@ -144,17 +173,18 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
|
|||
...overrides,
|
||||
});
|
||||
|
||||
const keyEntry = (
|
||||
api_key_id: string,
|
||||
overrides: Partial<ShadowEvalJob["keys"][number]> = {},
|
||||
): ShadowEvalJob["keys"][number] => ({
|
||||
api_key_id,
|
||||
const targetEntry = (
|
||||
target_id: string,
|
||||
overrides: Partial<ShadowEvalJob["targets"][number]> = {},
|
||||
): ShadowEvalJob["targets"][number] => ({
|
||||
target_type: "key",
|
||||
target_id,
|
||||
max_turns: 10000,
|
||||
max_budget: 10,
|
||||
spend: 0,
|
||||
stopped_at: null,
|
||||
attempt_count: null,
|
||||
key_alias: null,
|
||||
target_alias: null,
|
||||
key_name: null,
|
||||
...overrides,
|
||||
});
|
||||
|
|
@ -235,8 +265,8 @@ describe("ShadowEvalSection", () => {
|
|||
it("gives every active job its own card with a stop button, with the form still offered", () => {
|
||||
mockHooks({
|
||||
jobs: [
|
||||
job({ job_id: "job-a", status: "running", keys: [keyEntry("key-a")] }),
|
||||
job({ job_id: "job-b", status: "running", keys: [keyEntry("key-b")] }),
|
||||
job({ job_id: "job-a", status: "running", targets: [targetEntry("key-a")] }),
|
||||
job({ job_id: "job-b", status: "running", targets: [targetEntry("key-b")] }),
|
||||
],
|
||||
});
|
||||
render(<ShadowEvalSection />);
|
||||
|
|
@ -347,7 +377,7 @@ describe("ShadowEvalSection", () => {
|
|||
});
|
||||
|
||||
it("shows spend without a budget cap for a job from before spend budgets existed", () => {
|
||||
const j = job({ keys: [keyEntry("hashed-key-abc", { max_budget: null, spend: 3.21 })] });
|
||||
const j = job({ targets: [targetEntry("hashed-key-abc", { max_budget: null, spend: 3.21 })] });
|
||||
mockHooks({ jobs: [j], detailsById: { "job-1": j } });
|
||||
render(<ShadowEvalSection />);
|
||||
expect(screen.getByText(/\$3\.21 eval spend/)).toBeInTheDocument();
|
||||
|
|
@ -417,6 +447,38 @@ describe("ShadowEvalSection", () => {
|
|||
|
||||
const expectedBody = {
|
||||
api_key_ids: ["hash-alpha", "hash-beta"],
|
||||
team_ids: [],
|
||||
user_ids: [],
|
||||
router_name: "gpt-auto",
|
||||
direction: "forward",
|
||||
shadow_percentage: 10,
|
||||
duration_days: 7,
|
||||
max_budget: 10,
|
||||
judge_model: "anthropic/claude-sonnet-5",
|
||||
};
|
||||
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
|
||||
});
|
||||
|
||||
it("submits a team-only job with team_ids and no keys", async () => {
|
||||
const user = userEvent.setup();
|
||||
const { start } = mockHooks({});
|
||||
render(<ShadowEvalSection />);
|
||||
|
||||
expect(screen.getByText("Start shadow eval")).toBeDisabled();
|
||||
|
||||
await user.click(screen.getByPlaceholderText("Search teams by alias"));
|
||||
const teamList = await screen.findByTestId("paginated-multi-select-list");
|
||||
await user.click(within(teamList).getByText("engineering"));
|
||||
await user.click(screen.getByPlaceholderText("Select an auto-router"));
|
||||
await user.click(await screen.findByText("gpt-auto"));
|
||||
await user.click(screen.getByPlaceholderText("Select a judge model"));
|
||||
await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ }));
|
||||
await user.click(screen.getByText("Start shadow eval"));
|
||||
|
||||
const expectedBody = {
|
||||
api_key_ids: [],
|
||||
team_ids: ["team-eng"],
|
||||
user_ids: [],
|
||||
router_name: "gpt-auto",
|
||||
direction: "forward",
|
||||
shadow_percentage: 10,
|
||||
|
|
@ -453,6 +515,8 @@ describe("ShadowEvalSection", () => {
|
|||
|
||||
const expectedBody = {
|
||||
api_key_ids: ["hash-alpha"],
|
||||
team_ids: [],
|
||||
user_ids: [],
|
||||
router_name: "gpt-auto",
|
||||
direction: "reverse",
|
||||
baseline_model: "prod-claude",
|
||||
|
|
@ -485,9 +549,13 @@ describe("ShadowEvalSection", () => {
|
|||
});
|
||||
|
||||
it("labels the shadowed key by alias, then masked name, then truncated hash", () => {
|
||||
expect(shadowedKeyLabel(job().keys[0])).toBe("prod-alpha");
|
||||
expect(shadowedKeyLabel(keyEntry("hashed-key-abc", { key_name: "sk-...alpha" }))).toBe("sk-...alpha");
|
||||
expect(shadowedKeyLabel(keyEntry("hashed-key-abc"))).toBe("hashed-key…");
|
||||
expect(shadowedTargetLabel(job().targets[0])).toBe("prod-alpha");
|
||||
expect(shadowedTargetLabel(targetEntry("hashed-key-abc", { key_name: "sk-...alpha" }))).toBe("sk-...alpha");
|
||||
expect(shadowedTargetLabel(targetEntry("hashed-key-abc"))).toBe("hashed-key…");
|
||||
expect(shadowedTargetLabel(targetEntry("team-eng", { target_type: "team" }))).toBe("team-eng");
|
||||
expect(shadowedTargetLabel(targetEntry("team-eng", { target_type: "team", target_alias: "engineering" }))).toBe(
|
||||
"engineering",
|
||||
);
|
||||
});
|
||||
|
||||
it("breaks results down per key, so one key exhausting its own budget is visible while a sibling runs on", () => {
|
||||
|
|
@ -495,15 +563,12 @@ describe("ShadowEvalSection", () => {
|
|||
jobs: [
|
||||
job({
|
||||
judged_count: 205,
|
||||
keys: [
|
||||
keyEntry("hash-spent", { max_budget: 2, spend: 1.5, stopped_at: "2026-08-08T00:00:00Z" }),
|
||||
keyEntry("hash-hungry", { max_budget: 5, spend: 0.2 }),
|
||||
],
|
||||
results: {
|
||||
by_tier: [],
|
||||
by_current_model: [],
|
||||
by_key: [
|
||||
{
|
||||
targets: [
|
||||
targetEntry("hash-spent", {
|
||||
max_budget: 2,
|
||||
spend: 1.5,
|
||||
stopped_at: "2026-08-08T00:00:00Z",
|
||||
verdicts: {
|
||||
group: "hash-spent",
|
||||
turn_count: 200,
|
||||
real_win_rate_pct: 20.0,
|
||||
|
|
@ -514,7 +579,12 @@ describe("ShadowEvalSection", () => {
|
|||
shadow_spend: 0.5,
|
||||
cache_hit_turns: 0,
|
||||
},
|
||||
],
|
||||
}),
|
||||
targetEntry("hash-hungry", { max_budget: 5, spend: 0.2 }),
|
||||
],
|
||||
results: {
|
||||
by_tier: [],
|
||||
by_current_model: [],
|
||||
overall_shadow_win_rate_pct: 60.0,
|
||||
overall_tie_rate_pct: 20.0,
|
||||
sampled_real_spend: 0.9,
|
||||
|
|
@ -539,7 +609,7 @@ describe("ShadowEvalSection", () => {
|
|||
|
||||
expect(screen.getByText(/205 turns judged/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/Shadowing 10% of/)).toBeInTheDocument();
|
||||
expect(screen.getByText("2 keys")).toBeInTheDocument();
|
||||
expect(screen.getByText("2 targets")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("reads a key that spent its budget as completed even before the sweep stamps it", () => {
|
||||
|
|
@ -547,9 +617,9 @@ describe("ShadowEvalSection", () => {
|
|||
mockHooks({
|
||||
jobs: [
|
||||
job({
|
||||
keys: [
|
||||
keyEntry("hash-spent", { max_budget: 2, spend: 2, attempt_count: 40 }),
|
||||
keyEntry("hash-hungry", legacyTurnBudgetLeg),
|
||||
targets: [
|
||||
targetEntry("hash-spent", { max_budget: 2, spend: 2, attempt_count: 40 }),
|
||||
targetEntry("hash-hungry", legacyTurnBudgetLeg),
|
||||
],
|
||||
}),
|
||||
],
|
||||
|
|
@ -571,9 +641,9 @@ describe("ShadowEvalSection", () => {
|
|||
job({
|
||||
judged_count: 0,
|
||||
results: null,
|
||||
keys: [
|
||||
keyEntry("hash-spent", { max_budget: 0.5, spend: 0.5, attempt_count: 2 }),
|
||||
keyEntry("hash-hungry", { max_budget: 5, spend: 0.01, attempt_count: 1 }),
|
||||
targets: [
|
||||
targetEntry("hash-spent", { max_budget: 0.5, spend: 0.5, attempt_count: 2 }),
|
||||
targetEntry("hash-hungry", { max_budget: 5, spend: 0.01, attempt_count: 1 }),
|
||||
],
|
||||
}),
|
||||
],
|
||||
|
|
@ -594,9 +664,9 @@ describe("ShadowEvalSection", () => {
|
|||
jobs: [
|
||||
job({
|
||||
status: "completed",
|
||||
keys: [
|
||||
keyEntry("hash-spent", { max_turns: 200, stopped_at: "2026-08-08T00:00:00Z" }),
|
||||
keyEntry("hash-hungry", { max_turns: 500 }),
|
||||
targets: [
|
||||
targetEntry("hash-spent", { max_turns: 200, stopped_at: "2026-08-08T00:00:00Z" }),
|
||||
targetEntry("hash-hungry", { max_turns: 500 }),
|
||||
],
|
||||
}),
|
||||
],
|
||||
|
|
|
|||
|
|
@ -3,10 +3,13 @@
|
|||
import React, { useMemo, useState } from "react";
|
||||
|
||||
import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys";
|
||||
import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers";
|
||||
import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
|
||||
import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap";
|
||||
import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels";
|
||||
import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect";
|
||||
import TeamMultiSelect from "@/components/common_components/team_multi_select";
|
||||
import { userOptionLabel } from "@/components/common_components/UserDropdown";
|
||||
import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
|
@ -27,7 +30,7 @@ import {
|
|||
useStartShadowEval,
|
||||
useStopShadowEval,
|
||||
type ShadowEvalJob,
|
||||
type ShadowEvalJobKey,
|
||||
type ShadowEvalJobTarget,
|
||||
type ShadowEvalSlice,
|
||||
} from "./useShadowEval";
|
||||
|
||||
|
|
@ -66,29 +69,31 @@ const routerMatchedOrBeatPct = (
|
|||
? 100 - results.overall_shadow_win_rate_pct
|
||||
: results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct;
|
||||
|
||||
export const shadowedKeyLabel = (key: ShadowEvalJobKey): string =>
|
||||
key.key_alias || key.key_name || `${key.api_key_id.slice(0, 10)}…`;
|
||||
export const shadowedTargetLabel = (target: ShadowEvalJobTarget): string =>
|
||||
target.target_alias ||
|
||||
target.key_name ||
|
||||
(target.target_type === "key" ? `${target.target_id.slice(0, 10)}…` : target.target_id);
|
||||
|
||||
const shadowedKeysLabel = (job: ShadowEvalJob): string =>
|
||||
job.keys.length === 1 ? shadowedKeyLabel(job.keys[0]) : `${job.keys.length} keys`;
|
||||
const shadowedTargetsLabel = (job: ShadowEvalJob): string =>
|
||||
job.targets.length === 1 ? shadowedTargetLabel(job.targets[0]) : `${job.targets.length} targets`;
|
||||
|
||||
const totalBudget = (job: ShadowEvalJob): number | null =>
|
||||
job.keys.reduce<number | null>(
|
||||
(sum, key) => (sum === null || key.max_budget == null ? null : sum + key.max_budget),
|
||||
job.targets.reduce<number | null>(
|
||||
(sum, target) => (sum === null || target.max_budget == null ? null : sum + target.max_budget),
|
||||
0,
|
||||
);
|
||||
|
||||
const totalSpend = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + (key.spend ?? 0), 0);
|
||||
const totalSpend = (job: ShadowEvalJob): number => job.targets.reduce((sum, target) => sum + (target.spend ?? 0), 0);
|
||||
|
||||
const keySpent = (key: ShadowEvalJobKey): boolean => {
|
||||
const spendBudgetReached = key.max_budget != null && key.spend != null && key.spend >= key.max_budget;
|
||||
const turnValveReached = key.attempt_count != null && key.attempt_count >= key.max_turns;
|
||||
const targetSpent = (target: ShadowEvalJobTarget): boolean => {
|
||||
const spendBudgetReached = target.max_budget != null && target.spend != null && target.spend >= target.max_budget;
|
||||
const turnValveReached = target.attempt_count != null && target.attempt_count >= target.max_turns;
|
||||
return spendBudgetReached || turnValveReached;
|
||||
};
|
||||
|
||||
const keyStatus = (job: ShadowEvalJob, key: ShadowEvalJobKey): string => {
|
||||
if (job.status === "completed" || (key.stopped_at == null && keySpent(key))) return "completed";
|
||||
return key.stopped_at != null ? "stopped" : "running";
|
||||
const targetStatus = (job: ShadowEvalJob, target: ShadowEvalJobTarget): string => {
|
||||
if (job.status === "completed" || (target.stopped_at == null && targetSpent(target))) return "completed";
|
||||
return target.stopped_at != null ? "stopped" : "running";
|
||||
};
|
||||
|
||||
const jobHeadline = (job: ShadowEvalJob): React.ReactNode =>
|
||||
|
|
@ -96,12 +101,12 @@ const jobHeadline = (job: ShadowEvalJob): React.ReactNode =>
|
|||
<>
|
||||
Comparing <span className="font-mono text-xs">{job.router_name}</span> to{" "}
|
||||
<span className="font-mono text-xs">{job.baseline_model}</span> on {job.shadow_percentage}% of{" "}
|
||||
<span className="font-mono text-xs">{shadowedKeysLabel(job)}</span> traffic
|
||||
<span className="font-mono text-xs">{shadowedTargetsLabel(job)}</span> traffic
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
Shadowing {job.shadow_percentage}% of <span className="font-mono text-xs">{shadowedKeysLabel(job)}</span> traffic
|
||||
via <span className="font-mono text-xs">{job.router_name}</span>
|
||||
Shadowing {job.shadow_percentage}% of <span className="font-mono text-xs">{shadowedTargetsLabel(job)}</span>{" "}
|
||||
traffic via <span className="font-mono text-xs">{job.router_name}</span>
|
||||
</>
|
||||
);
|
||||
|
||||
|
|
@ -255,13 +260,12 @@ const VerdictBar: React.FC<{ direction: ShadowEvalDirection; results: NonNullabl
|
|||
);
|
||||
};
|
||||
|
||||
const KeyTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => {
|
||||
const slices = new Map((job.results?.by_key ?? []).map((slice) => [slice.group, slice]));
|
||||
const TargetTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => {
|
||||
return (
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Key</TableHead>
|
||||
<TableHead>Target</TableHead>
|
||||
<TableHead>Status</TableHead>
|
||||
{["Budget used", "Router wins", `${otherArmLabel(job.direction)} wins`].map((label) => (
|
||||
<TableHead key={label} className="text-right">
|
||||
|
|
@ -271,18 +275,23 @@ const KeyTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => {
|
|||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{job.keys.map((key) => {
|
||||
const slice = slices.get(key.api_key_id);
|
||||
{job.targets.map((target) => {
|
||||
const slice = target.verdicts;
|
||||
return (
|
||||
<TableRow key={key.api_key_id}>
|
||||
<TableCell className="font-medium text-foreground">{shadowedKeyLabel(key)}</TableCell>
|
||||
<TableRow key={`${target.target_type}:${target.target_id}`}>
|
||||
<TableCell className="font-medium text-foreground">
|
||||
{shadowedTargetLabel(target)}
|
||||
{target.target_type !== "key" && (
|
||||
<span className="ml-2 text-xs font-normal text-muted-foreground">{target.target_type}</span>
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<StatusBadge status={keyStatus(job, key)} />
|
||||
<StatusBadge status={targetStatus(job, target)} />
|
||||
</TableCell>
|
||||
<TableCell className="text-right tabular-nums">
|
||||
{key.max_budget != null
|
||||
? `${usd(key.spend ?? 0)} / ${usd(key.max_budget)}`
|
||||
: `${(key.attempt_count ?? slice?.turn_count ?? 0).toLocaleString()} / ${key.max_turns.toLocaleString()} turns`}
|
||||
{target.max_budget != null
|
||||
? `${usd(target.spend ?? 0)} / ${usd(target.max_budget)}`
|
||||
: `${(target.attempt_count ?? slice?.turn_count ?? 0).toLocaleString()} / ${target.max_turns.toLocaleString()} turns`}
|
||||
</TableCell>
|
||||
{slice ? (
|
||||
<>
|
||||
|
|
@ -318,9 +327,9 @@ const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({
|
|||
const hasVerdicts = results != null && (results.by_tier.length > 0 || results.by_current_model.length > 0);
|
||||
return (
|
||||
<>
|
||||
{job.keys.length > 1 && (
|
||||
{job.targets.length > 1 && (
|
||||
<div className="border-b">
|
||||
<KeyTable job={job} />
|
||||
<TargetTable job={job} />
|
||||
</div>
|
||||
)}
|
||||
{/* results == null re-stated for TS narrowing; hasVerdicts alone cannot narrow it */}
|
||||
|
|
@ -454,9 +463,9 @@ const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[
|
|||
|
||||
const START_FORM_DESCRIPTION: Record<ShadowEvalDirection, string> = {
|
||||
forward:
|
||||
"Duplicates a sampled slice of the selected keys' traffic through the auto-router and has an LLM judge compare both answers blind. Each key gets its own spend budget. The router's answers are never served to users; judge calls bill to the shadowed key.",
|
||||
"Duplicates a sampled slice of the selected targets' traffic (keys, teams, or users) through the auto-router and has an LLM judge compare both answers blind. Each target gets its own spend budget. The router's answers are never served to users; judge calls bill to the sampled traffic's own identity.",
|
||||
reverse:
|
||||
"Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each key gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the shadowed key.",
|
||||
"Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each target gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the sampled traffic's own identity.",
|
||||
};
|
||||
|
||||
const DURATION_OPTIONS = [
|
||||
|
|
@ -515,9 +524,46 @@ const KeySelect: React.FC<{ value: string[]; onChange: (tokens: string[]) => voi
|
|||
);
|
||||
};
|
||||
|
||||
const UserSelect: React.FC<{ value: string[]; onChange: (ids: string[]) => void }> = ({ value, onChange }) => {
|
||||
const [search, setSearch] = useState("");
|
||||
const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteUsers(
|
||||
50,
|
||||
search || undefined,
|
||||
);
|
||||
const options = useMemo<SearchSelectOption[]>(
|
||||
() =>
|
||||
Array.from(
|
||||
new Map(
|
||||
(data?.pages ?? [])
|
||||
.flatMap((page) => page.users)
|
||||
.map((user) => [user.user_id, { label: userOptionLabel(user), value: user.user_id }] as const),
|
||||
).values(),
|
||||
),
|
||||
[data],
|
||||
);
|
||||
return (
|
||||
<PaginatedMultiSelect
|
||||
inputId="shadow-eval-user"
|
||||
options={options}
|
||||
value={value}
|
||||
onValueChange={onChange}
|
||||
onSearchChange={setSearch}
|
||||
onLoadMore={() => void fetchNextPage()}
|
||||
hasNextPage={hasNextPage}
|
||||
isFetchingNextPage={isFetchingNextPage}
|
||||
isLoading={isPending}
|
||||
placeholder="Search users by email"
|
||||
emptyText="No matching users"
|
||||
errorText={isError ? "Users could not be loaded. Refresh the page to retry." : undefined}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const StartForm: React.FC = () => {
|
||||
const { accessToken } = useAuthorized();
|
||||
const [apiKeyIds, setApiKeyIds] = useState<string[]>([]);
|
||||
const [teamIds, setTeamIds] = useState<string[]>([]);
|
||||
const [userIds, setUserIds] = useState<string[]>([]);
|
||||
const [routerName, setRouterName] = useState("");
|
||||
const [direction, setDirection] = useState<ShadowEvalDirection>("forward");
|
||||
const [baselineModel, setBaselineModel] = useState("");
|
||||
|
|
@ -542,12 +588,15 @@ const StartForm: React.FC = () => {
|
|||
const parsedMaxBudget = Number.parseFloat(maxBudget);
|
||||
const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000;
|
||||
const baselinePicked = direction === "forward" || baselineModel !== "";
|
||||
const filled = apiKeyIds.length > 0 && [routerName, judgeModel].every((field) => field !== "") && baselinePicked;
|
||||
const targetsPicked = apiKeyIds.length + teamIds.length + userIds.length > 0;
|
||||
const filled = targetsPicked && [routerName, judgeModel].every((field) => field !== "") && baselinePicked;
|
||||
const boundsValid = percentageValid && maxBudgetValid;
|
||||
const valid = Boolean(accessToken) && filled && boundsValid;
|
||||
const handleStart = () => {
|
||||
const startBody = {
|
||||
api_key_ids: apiKeyIds,
|
||||
team_ids: teamIds,
|
||||
user_ids: userIds,
|
||||
router_name: routerName,
|
||||
direction,
|
||||
...(direction === "reverse" ? { baseline_model: baselineModel } : {}),
|
||||
|
|
@ -587,6 +636,12 @@ const StartForm: React.FC = () => {
|
|||
<Field label="Keys to shadow" htmlFor="shadow-eval-key">
|
||||
<KeySelect value={apiKeyIds} onChange={setApiKeyIds} />
|
||||
</Field>
|
||||
<Field label="Teams to shadow">
|
||||
<TeamMultiSelect value={teamIds} onChange={setTeamIds} placeholder="Search teams by alias" />
|
||||
</Field>
|
||||
<Field label="Users to shadow" htmlFor="shadow-eval-user">
|
||||
<UserSelect value={userIds} onChange={setUserIds} />
|
||||
</Field>
|
||||
<Field label="Auto-router">
|
||||
<SearchSelect
|
||||
options={routerOptions}
|
||||
|
|
@ -642,7 +697,7 @@ const StartForm: React.FC = () => {
|
|||
value={maxBudget}
|
||||
onChange={(e) => setMaxBudget(e.target.value)}
|
||||
/>
|
||||
<span className="text-sm text-muted-foreground">max shadow + judge spend, per key</span>
|
||||
<span className="text-sm text-muted-foreground">max shadow + judge spend, per target</span>
|
||||
</div>
|
||||
{maxBudget.trim() !== "" && !maxBudgetValid && (
|
||||
<p className="text-xs text-destructive">Enter a value from 0.01 to 10000</p>
|
||||
|
|
@ -774,8 +829,9 @@ const ShadowEvalSection: React.FC = () => {
|
|||
<div className="flex flex-wrap items-baseline gap-2">
|
||||
<h2 className="text-xl font-semibold text-foreground">Shadow eval</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Blind-judge the auto-router on your real traffic: against the models a key uses today before switching, or
|
||||
against a fixed baseline after it has switched.
|
||||
Blind-judge the auto-router on the real traffic of a key, team, or user (teams and users cover
|
||||
JWT-authenticated traffic): against the models they use today before switching, or against a fixed baseline
|
||||
after they have switched.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import { $api, fetchClient } from "@/lib/http/api";
|
|||
import type { components } from "@/lib/http/schema";
|
||||
|
||||
export type ShadowEvalJob = components["schemas"]["ShadowEvalJobResponse"];
|
||||
export type ShadowEvalJobKey = components["schemas"]["ShadowEvalJobKeyResponse"];
|
||||
export type ShadowEvalJobTarget = components["schemas"]["ShadowEvalJobTargetResponse"];
|
||||
export type ShadowEvalSlice = components["schemas"]["ShadowEvalSlice"];
|
||||
export type StartShadowEvalRequest = components["schemas"]["StartShadowEvalRequest"];
|
||||
|
||||
|
|
|
|||
203
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
203
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -1234,8 +1234,8 @@ export interface paths {
|
|||
};
|
||||
/**
|
||||
* List Shadow Eval Jobs
|
||||
* @description List shadow eval jobs, newest first, each key with its attempt count so status is
|
||||
* accurate. Judged counts, spend, and results ride the detail endpoint only.
|
||||
* @description List shadow eval jobs, newest first, each target with its attempt count so status
|
||||
* is accurate. Judged counts, spend, and results ride the detail endpoint only.
|
||||
*/
|
||||
get: operations["list_shadow_eval_jobs_auto_router_shadow_eval_get"];
|
||||
put?: never;
|
||||
|
|
@ -1257,22 +1257,29 @@ export interface paths {
|
|||
put?: never;
|
||||
/**
|
||||
* Start Shadow Eval
|
||||
* @description Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against
|
||||
* a second arm, judge the two responses blind, and stratify win rates by tier, by the model
|
||||
* that served the real arm, and by key.
|
||||
* @description Start a shadow eval: duplicate a sampled slice of one or more targets' live traffic
|
||||
* against a second arm, judge the two responses blind, and stratify win rates by tier,
|
||||
* by the model that served the real arm, and by target.
|
||||
*
|
||||
* A forward job answers whether the keys should adopt router_name: it samples the requests
|
||||
* the router did not serve and duplicates them through it. A reverse job answers whether a
|
||||
* key already on the router still gains from it: it samples the requests the router did
|
||||
* serve and duplicates them against baseline_model. A key can hold one active job per
|
||||
* direction, so both questions can run at once.
|
||||
* A target is a virtual key, a team, or a user. Team and user targets match on the
|
||||
* identity every request resolves to at auth time, so they cover JWT-authenticated
|
||||
* traffic, which presents no virtual key; a user target samples that user's traffic
|
||||
* across all their teams, whether it arrives on a JWT or a key they own.
|
||||
*
|
||||
* Shadow responses are never served to users. Each key samples until its recorded eval
|
||||
* spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's
|
||||
* window ends, or the job is stopped, so one key running out of budget does not end
|
||||
* sampling for the others; sampling changes propagate to pods within about 10 seconds.
|
||||
* Shadow and judge calls bill to the shadowed key but are excluded from request counts
|
||||
* and auto-router adoption metrics.
|
||||
* A forward job answers whether the targets should adopt router_name: it samples the
|
||||
* requests the router did not serve and duplicates them through it. A reverse job
|
||||
* answers whether a target already on the router still gains from it: it samples the
|
||||
* requests the router did serve and duplicates them against baseline_model. A target
|
||||
* can hold one active job per direction, so both questions can run at once, and a
|
||||
* request matching several jobs' targets (say its key and its team) is sampled by
|
||||
* each, separately budgeted.
|
||||
*
|
||||
* Shadow responses are never served to users. Each target samples until its recorded
|
||||
* eval spend, the shadow and judge calls' own cost, reaches max_budget dollars, the
|
||||
* job's window ends, or the job is stopped, so one target running out of budget does
|
||||
* not end sampling for the others; sampling changes propagate to pods within about 10
|
||||
* seconds. Shadow and judge calls bill to the sampled request's own identity but are
|
||||
* excluded from request counts and auto-router adoption metrics.
|
||||
*/
|
||||
post: operations["start_shadow_eval_auto_router_shadow_eval_start_post"];
|
||||
delete?: never;
|
||||
|
|
@ -1312,8 +1319,8 @@ export interface paths {
|
|||
put?: never;
|
||||
/**
|
||||
* Stop Shadow Eval Job
|
||||
* @description Stop an active shadow eval job, every key it scopes at once. Attempts are kept;
|
||||
* sampling halts within ~10s. Keys that already stopped on their own budget keep the
|
||||
* @description Stop an active shadow eval job, every target it scopes at once. Attempts are kept;
|
||||
* sampling halts within ~10s. Targets that already stopped on their own budget keep the
|
||||
* stopped_at they earned. The statement is the whole state machine: it claims the job
|
||||
* only while a leg still samples inside the window with no stop recorded, so a racing
|
||||
* operator, a same-instant budget spend, and a repeat stop all read the same 400 with
|
||||
|
|
@ -35262,56 +35269,10 @@ export interface components {
|
|||
/** Timeout */
|
||||
timeout?: number | null;
|
||||
};
|
||||
/**
|
||||
* ShadowEvalJobKeyResponse
|
||||
* @description One key a job shadows, with its own budget and stop state.
|
||||
*/
|
||||
ShadowEvalJobKeyResponse: {
|
||||
/**
|
||||
* Api Key Id
|
||||
* @description The hashed virtual key whose traffic this entry scopes
|
||||
*/
|
||||
api_key_id: string;
|
||||
/**
|
||||
* Attempt Count
|
||||
* @description This key's sampled attempts so far, judged and errored alike, the same count the sampler budgets against max_turns; populated on list and detail responses. Frozen at stopped_at once the key is stamped, so in-flight attempts landing after a stop never reclassify it
|
||||
*/
|
||||
attempt_count?: number | null;
|
||||
/**
|
||||
* Key Alias
|
||||
* @description Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted
|
||||
*/
|
||||
key_alias?: string | null;
|
||||
/**
|
||||
* Key Name
|
||||
* @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias
|
||||
*/
|
||||
key_name?: string | null;
|
||||
/**
|
||||
* Max Budget
|
||||
* @description This key's own USD budget for the eval's shadow and judge spend, independent of its siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds
|
||||
*/
|
||||
max_budget?: number | null;
|
||||
/**
|
||||
* Max Turns
|
||||
* @description This key's sample-count ceiling: the whole budget for jobs created before max_budget existed, and the error-loop safety valve otherwise
|
||||
*/
|
||||
max_turns: number;
|
||||
/**
|
||||
* Spend
|
||||
* @description This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets against max_budget; populated on list and detail responses and frozen at stopped_at exactly like attempt_count
|
||||
*/
|
||||
spend?: number | null;
|
||||
/**
|
||||
* Stopped At
|
||||
* @description When this key's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset
|
||||
*/
|
||||
stopped_at?: string | null;
|
||||
};
|
||||
/**
|
||||
* ShadowEvalJobResponse
|
||||
* @description A shadow-eval job over one or more keys, each with its own budget and stop state;
|
||||
* status is derived from stopped_by, the keys' stop and budget state, and ends_at,
|
||||
* @description A shadow-eval job over one or more targets, each with its own budget and stop state;
|
||||
* status is derived from stopped_by, the targets' stop and budget state, and ends_at,
|
||||
* never stored, so no writer anywhere can produce an inconsistent one. Aggregate
|
||||
* fields are populated by the detail endpoint only and stay None on list responses.
|
||||
*/
|
||||
|
|
@ -35353,11 +35314,6 @@ export interface components {
|
|||
* @description Verdicts recorded; detail endpoint only
|
||||
*/
|
||||
judged_count?: number | null;
|
||||
/**
|
||||
* Keys
|
||||
* @description The keys whose traffic this job evaluates, and only those keys', each with its own budget
|
||||
*/
|
||||
keys: components["schemas"]["ShadowEvalJobKeyResponse"][];
|
||||
/**
|
||||
* Last Error
|
||||
* @description Most recent attempt error; detail endpoint only
|
||||
|
|
@ -35373,8 +35329,8 @@ export interface components {
|
|||
* Status
|
||||
* @description Three recorded facts, no history-guessing: a stop is stopped_by (the migration
|
||||
* backfills it for every job that displayed stopped when the column arrived, so the
|
||||
* pre-column population is closed), completion is the window passing or every key
|
||||
* spending its budget, and anything else is running. The all-keys-stamped fallback
|
||||
* pre-column population is closed), completion is the window passing or every target
|
||||
* spending its budget, and anything else is running. The all-targets-stamped fallback
|
||||
* covers only stops written by pre-column pods during a rolling deploy.
|
||||
* @enum {string}
|
||||
*/
|
||||
|
|
@ -35384,6 +35340,65 @@ export interface components {
|
|||
* @description The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled by migration for jobs that displayed stopped when the column arrived; None when the job ended on its own. Its presence is what makes a job read stopped rather than completed
|
||||
*/
|
||||
stopped_by?: string | null;
|
||||
/**
|
||||
* Targets
|
||||
* @description The targets whose traffic this job evaluates, and only theirs, each with its own budget
|
||||
*/
|
||||
targets: components["schemas"]["ShadowEvalJobTargetResponse"][];
|
||||
};
|
||||
/**
|
||||
* ShadowEvalJobTargetResponse
|
||||
* @description One target a job shadows (a key, team, or user), with its own budget and stop state.
|
||||
*/
|
||||
ShadowEvalJobTargetResponse: {
|
||||
/**
|
||||
* Attempt Count
|
||||
* @description This target's sampled attempts so far, judged and errored alike, the same count the sampler budgets against max_turns; populated on list and detail responses. Frozen at stopped_at once the target is stamped, so in-flight attempts landing after a stop never reclassify it
|
||||
*/
|
||||
attempt_count?: number | null;
|
||||
/**
|
||||
* Key Name
|
||||
* @description Masked display name (sk-...) for key targets, resolved at read time; None for teams and users
|
||||
*/
|
||||
key_name?: string | null;
|
||||
/**
|
||||
* Max Budget
|
||||
* @description This target's own USD budget for the eval's shadow and judge spend, independent of its siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds
|
||||
*/
|
||||
max_budget?: number | null;
|
||||
/**
|
||||
* Max Turns
|
||||
* @description This target's sample-count ceiling: the whole budget for jobs created before max_budget existed, and the error-loop safety valve otherwise
|
||||
*/
|
||||
max_turns: number;
|
||||
/**
|
||||
* Spend
|
||||
* @description This target's recorded shadow plus judge spend in USD, the same figure the sampler budgets against max_budget; populated on list and detail responses and frozen at stopped_at exactly like attempt_count
|
||||
*/
|
||||
spend?: number | null;
|
||||
/**
|
||||
* Stopped At
|
||||
* @description When this target's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset
|
||||
*/
|
||||
stopped_at?: string | null;
|
||||
/**
|
||||
* Target Alias
|
||||
* @description Display label resolved from the target's own row at read time: the key's alias, the team's alias, or the user's email; None when unset or deleted
|
||||
*/
|
||||
target_alias?: string | null;
|
||||
/**
|
||||
* Target Id
|
||||
* @description The hashed virtual key, team id, or user id whose traffic this entry scopes
|
||||
*/
|
||||
target_id: string;
|
||||
/**
|
||||
* Target Type
|
||||
* @description What kind of entity this entry scopes
|
||||
* @enum {string}
|
||||
*/
|
||||
target_type: "key" | "team" | "user";
|
||||
/** @description This target's own judged-verdict slice; detail endpoint only, None until a turn is judged */
|
||||
verdicts?: components["schemas"]["ShadowEvalSlice"] | null;
|
||||
};
|
||||
/**
|
||||
* ShadowEvalResult
|
||||
|
|
@ -35395,11 +35410,6 @@ export interface components {
|
|||
* @description Sliced by the model that served the real arm: the keys' incumbent models in forward mode, and in reverse the models the router itself picked
|
||||
*/
|
||||
by_current_model: components["schemas"]["ShadowEvalSlice"][];
|
||||
/**
|
||||
* By Key
|
||||
* @description One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job scopes but has not judged a turn for yet are absent rather than reported as zero
|
||||
*/
|
||||
by_key: components["schemas"]["ShadowEvalSlice"][];
|
||||
/** By Tier */
|
||||
by_tier: components["schemas"]["ShadowEvalSlice"][];
|
||||
/**
|
||||
|
|
@ -35441,8 +35451,9 @@ export interface components {
|
|||
};
|
||||
/**
|
||||
* ShadowEvalSlice
|
||||
* @description Judge outcomes for one slice of a job's verdicts (a router tier, or one of the
|
||||
* models that served the real arm).
|
||||
* @description Judge outcomes for one slice of a job's verdicts: a router tier, one of the
|
||||
* models that served the real arm, or one scoped target (embedded on that target's
|
||||
* own entry, so slices never need re-joining to a target by id).
|
||||
*/
|
||||
ShadowEvalSlice: {
|
||||
/** Avg Judge Confidence */
|
||||
|
|
@ -35672,12 +35683,18 @@ export interface components {
|
|||
};
|
||||
/**
|
||||
* StartShadowEvalRequest
|
||||
* @description Start duplicating one or more keys' traffic for blind comparison against an auto-router.
|
||||
* @description Start duplicating one or more targets' traffic for blind comparison against an auto-router.
|
||||
*
|
||||
* A target is a virtual key, a team, or a user; each becomes its own leg with its own
|
||||
* budget and stop state. Team and user targets match on the identity every request
|
||||
* carries after auth (user_api_key_team_id / user_api_key_user_id), so they cover
|
||||
* JWT-authenticated traffic, which presents no virtual key at all.
|
||||
*/
|
||||
StartShadowEvalRequest: {
|
||||
/**
|
||||
* Api Key Ids
|
||||
* @description The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these keys' traffic; requests made with any other key are not sampled. Each key carries its own max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 keys per job, which also bounds every read the job's endpoints make.
|
||||
* @description Hashed virtual keys whose traffic will be shadowed. Combined with team_ids and user_ids the job needs at least one target and at most 100, which also bounds every read the job's endpoints make. Each target carries its own max_budget spend budget, so one exhausting its budget leaves the others sampling.
|
||||
* @default []
|
||||
*/
|
||||
api_key_ids: string[];
|
||||
/**
|
||||
|
|
@ -35706,7 +35723,7 @@ export interface components {
|
|||
judge_model: string;
|
||||
/**
|
||||
* Max Budget
|
||||
* @description Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window
|
||||
* @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window
|
||||
* @default 10
|
||||
*/
|
||||
max_budget: number;
|
||||
|
|
@ -35717,9 +35734,21 @@ export interface components {
|
|||
router_name: string;
|
||||
/**
|
||||
* Shadow Percentage
|
||||
* @description Percentage of the key's requests to duplicate through the router
|
||||
* @description Percentage of each target's requests to duplicate through the router
|
||||
*/
|
||||
shadow_percentage: number;
|
||||
/**
|
||||
* Team Ids
|
||||
* @description Teams whose traffic will be shadowed, matched on the team every authenticated request resolves to, so a team's JWT-auth and virtual-key traffic are both sampled
|
||||
* @default []
|
||||
*/
|
||||
team_ids: string[];
|
||||
/**
|
||||
* User Ids
|
||||
* @description Users whose traffic will be shadowed, matched on the user every authenticated request resolves to across all their teams: JWT requests carrying their subject claim and virtual keys they own
|
||||
* @default []
|
||||
*/
|
||||
user_ids: string[];
|
||||
};
|
||||
/**
|
||||
* SuccessfulKeyUpdate
|
||||
|
|
@ -40681,8 +40710,10 @@ export interface operations {
|
|||
list_shadow_eval_jobs_auto_router_shadow_eval_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
/** @description Filter to jobs that shadow this key, alone or alongside others */
|
||||
api_key_id?: string | null;
|
||||
/** @description Kind of target to filter on; requires target_id */
|
||||
target_type?: ("key" | "team" | "user") | null;
|
||||
/** @description Filter to jobs that shadow this target, alone or alongside others */
|
||||
target_id?: string | null;
|
||||
/** @description Newest jobs to return */
|
||||
limit?: number;
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue