diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260831000000_shadow_eval_typed_targets/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260831000000_shadow_eval_typed_targets/migration.sql new file mode 100644 index 00000000000..b7dbe931dd2 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260831000000_shadow_eval_typed_targets/migration.sql @@ -0,0 +1,21 @@ +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'LiteLLM_ShadowEvalJob' AND column_name = 'api_key_id' + ) THEN + ALTER TABLE "LiteLLM_ShadowEvalJob" RENAME COLUMN "api_key_id" TO "target_id"; + END IF; +END $$; + +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "target_type" TEXT NOT NULL DEFAULT 'key'; + +DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key_direction"; + +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_one_active_per_target_direction" + ON "LiteLLM_ShadowEvalJob"("target_type", "target_id", "direction") WHERE "stopped_at" IS NULL; + +DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_api_key_id_idx"; + +CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_target_type_target_id_idx" + ON "LiteLLM_ShadowEvalJob"("target_type", "target_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 60223265211..01a607b68a9 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession { model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) group_id String // legs of one job share this; the API's job id - api_key_id String // hashed virtual key whose traffic this leg shadows + target_type String @default("key") // key | team | user + target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise - max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets + max_budget Float? // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets created_at DateTime @default(now()) created_by String? ends_at DateTime @@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob { stopped_by String? // operator who stopped it early; null when it ended on its own @@index([group_id]) - @@index([api_key_id]) + @@index([target_type, target_id]) @@index([created_at]) } diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index cf8aa38d86e..18bda0a9d55 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -592,7 +592,12 @@ _JOBS_CACHE_KEY: Final = "shadow_eval:active_jobs" class ShadowEvalLogger(CustomLogger): - """Fires blind pairwise shadow evaluations for keys with an active shadow-eval job.""" + """Fires blind pairwise shadow evaluations for targets with an active shadow-eval job. + + A job targets a virtual key, a team, or a user; a request qualifies for a job when + any of its resolved identities (key hash, team id, user id) matches the job's + target, so team and user jobs cover JWT-authenticated traffic, which carries no + key hash at all.""" def __init__( self, @@ -617,10 +622,10 @@ class ShadowEvalLogger(CustomLogger): # generation; the refill absorbs written rows and resets. self._job_starts: dict[str, int] = {} # mutable-ok: per-generation counter - async def _active_jobs(self) -> Mapping[str, tuple[ActiveShadowEvalJob, ...]]: - """Active jobs by api_key_id, cache-first. A key holds at most one job per - direction, so the value is a collection. A DB fault returns empty without - caching, so sampling pauses for that request and the next one retries.""" + async def _active_jobs(self) -> Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]: + """Active jobs by (target_type, target_id), cache-first. A target holds at most + one job per direction, so the value is a collection. A DB fault returns empty + without caching, so sampling pauses for that request and the next one retries.""" cached: Final = await self._jobs_cache.async_get_cache(_JOBS_CACHE_KEY) if cached is not None: return cached # pyright: ignore[reportReturnType] # cache stores exactly this mapping shape @@ -652,10 +657,10 @@ class ShadowEvalLogger(CustomLogger): ) for row in grouped or [] } - by_key: Final = tuple( + by_target: Final = tuple( sorted( ( - (str(record.api_key_id), job) + ((str(record.target_type), str(record.target_id)), job) for record in records or [] if (job := _as_active_job(record, *attempt_stats.get(str(record.id), (0, 0.0)))) is not None ), @@ -663,7 +668,7 @@ class ShadowEvalLogger(CustomLogger): ) ) jobs: Final = MappingProxyType( - {key: tuple(job for _, job in group) for key, group in groupby(by_key, key=itemgetter(0))} + {target: tuple(job for _, job in group) for target, group in groupby(by_target, key=itemgetter(0))} ) await self._jobs_cache.async_set_cache(_JOBS_CACHE_KEY, jobs) self._job_starts = {} # rebind-ok: new generation, counts absorbed into the fill @@ -720,8 +725,18 @@ class ShadowEvalLogger(CustomLogger): if should_redact_message_logging(dict(kwargs)): # mutable-ok: predicate takes a plain dict return metadata: Final = payload.get("metadata") or _EMPTY_METADATA - api_key_hash: Final = metadata.get("user_api_key_hash") - if not api_key_hash: + # Each identity the request resolved to is a candidate target; JWT-auth + # requests carry no key hash but do carry a team and user. + targets: Final = tuple( + (target_type, str(value)) + for target_type, value in ( + ("key", metadata.get("user_api_key_hash")), + ("team", metadata.get("user_api_key_team_id")), + ("user", metadata.get("user_api_key_user_id")), + ) + if value + ) + if not targets: return request_id: Final = payload.get("id") or "" if not request_id: @@ -731,8 +746,11 @@ class ShadowEvalLogger(CustomLogger): return # only surfaces this table can normalize are comparable; unknown types fail closed if ops.wire_params and _request_mutating_guardrail_ran(request_metadata): return # the wire-body snapshot predates the rewrite; replaying it would resurrect stripped content + active_jobs: Final = await self._active_jobs() eligible: Final = self._sampled_jobs( - (await self._active_jobs()).get(str(api_key_hash), ()), request_metadata, request_id + tuple(job for target in targets for job in active_jobs.get(target, ())), + request_metadata, + request_id, ) if not eligible: return @@ -1056,7 +1074,7 @@ class ShadowEvalLogger(CustomLogger): ) -_EMPTY_JOBS: Final[Mapping[str, tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({}) +_EMPTY_JOBS: Final[Mapping[tuple[str, str], tuple[ActiveShadowEvalJob, ...]]] = MappingProxyType({}) def _default_prisma_provider() -> "PrismaClient | None": diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index b5533d548e5..44b0cdcca2e 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -7,7 +7,7 @@ POST /auto_router/validate_complexity_router_config - Dry-run the complexity-rou from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone -from itertools import groupby +from itertools import chain, groupby from operator import attrgetter from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Protocol @@ -58,10 +58,11 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( ComplexityRouterConfigValidationResponse, RequestComplexityRouterConfig, ShadowEvalDirection, - ShadowEvalJobKeyResponse, ShadowEvalJobResponse, + ShadowEvalJobTargetResponse, ShadowEvalResult, ShadowEvalSlice, + ShadowEvalTargetType, StartShadowEvalRequest, ) @@ -104,10 +105,43 @@ class _VerificationTokenTable(Protocol): async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_VerificationTokenRow]: ... +class _TeamRow(Protocol): + @property + def team_id(self) -> str: ... + + @property + def team_alias(self) -> str | None: ... + + +class _TeamRowsTable(Protocol): + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_TeamRow]: ... + + +class _UserRow(Protocol): + @property + def user_id(self) -> str: ... + + @property + def user_email(self) -> str | None: ... + + +class _UserRowsTable(Protocol): + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_UserRow]: ... + + class _ShadowEvalJobRow(Protocol): @property def id(self) -> str: ... + @property + def group_id(self) -> str: ... + + @property + def target_type(self) -> str: ... + + @property + def target_id(self) -> str: ... + class _ShadowEvalJobTable(Protocol): async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ShadowEvalJobRow]: ... @@ -138,6 +172,14 @@ def _verification_tokens(prisma_client: "PrismaClient") -> _VerificationTokenTab return prisma_client.db.litellm_verificationtoken +def _team_rows(prisma_client: "PrismaClient") -> _TeamRowsTable: + return prisma_client.db.litellm_teamtable + + +def _user_rows(prisma_client: "PrismaClient") -> _UserRowsTable: + return prisma_client.db.litellm_usertable + + def _shadow_eval_jobs(prisma_client: "PrismaClient") -> _ShadowEvalJobTable: return prisma_client.db.litellm_shadowevaljob @@ -836,7 +878,7 @@ def _validate_judge_is_not_a_candidate( def _is_unique_violation(error: Exception) -> bool: - """Whether a Prisma create failed on a unique index. One active job per key and + """Whether a Prisma create failed on a unique index. One active job per target and direction lives in a partial unique index (raw SQL in the migration; schema.prisma cannot express partial indexes), so the read-then-create check above it is advisory: two concurrent starts pass the read, and the loser must surface as the same 409 @@ -885,7 +927,7 @@ _ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT # direction, and mid-deploy rows from old pods price as judge-only until the deploy ends). _SWEEP_FINISHED_JOBS_SQL: Final = """ UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = (NOW() AT TIME ZONE 'utc') -WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL +WHERE j.target_type = $2 AND j.target_id = ANY($1::text[]) AND j.stopped_at IS NULL AND ( j.ends_at <= (NOW() AT TIME ZONE 'utc') OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns @@ -966,10 +1008,10 @@ WHERE group_id IN ( ) """ -_LIST_LEGS_BY_KEY_SQL: Final = """ +_LIST_LEGS_BY_TARGET_SQL: Final = """ SELECT * FROM "LiteLLM_ShadowEvalJob" WHERE group_id IN ( - SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE api_key_id = $2 + SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE target_type = $2 AND target_id = $3 GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int ) """ @@ -1007,15 +1049,16 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]: class _LegRow(BaseModel): """One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is - one key's leg of a job; the legs of a job share group_id and identical config, written - together by one create_many. The API's job id is the group id, so leg ids never leave - the server (attempts reference them internally).""" + one target's leg of a job; the legs of a job share group_id and identical config, + written together by one create_many. The API's job id is the group id, so leg ids + never leave the server (attempts reference them internally).""" model_config = ConfigDict(from_attributes=True) id: str group_id: str - api_key_id: str + target_type: ShadowEvalTargetType + target_id: str router_name: str direction: ShadowEvalDirection baseline_model: str | None = None @@ -1068,16 +1111,17 @@ def _group_response( first: Final = legs[0] return ShadowEvalJobResponse( job_id=group_id, - keys=tuple( - ShadowEvalJobKeyResponse( - api_key_id=leg.api_key_id, + targets=tuple( + ShadowEvalJobTargetResponse( + target_type=leg.target_type, + target_id=leg.target_id, max_turns=leg.max_turns, max_budget=leg.max_budget, stopped_at=leg.stopped_at, attempt_count=stats.attempt_count if (stats := attempt_counts.get(leg.id)) else 0, spend=round(stats.spend, 6) if stats else 0.0, ) - for leg in sorted(legs, key=lambda leg: leg.api_key_id) + for leg in sorted(legs, key=lambda leg: (leg.target_type, leg.target_id)) ), router_name=first.router_name, direction=first.direction, @@ -1090,34 +1134,85 @@ def _group_response( ) -_NO_KEY_LABELS: Final[tuple[str | None, str | None]] = (None, None) +_NO_TARGET_LABELS: Final[tuple[str | None, str | None]] = (None, None) -async def _with_key_labels( +def _target_labels( + key_rows: Sequence[_VerificationTokenRow], + team_rows: Sequence[_TeamRow], + user_rows: Sequence[_UserRow], +) -> Mapping[tuple[str, str], tuple[str | None, str | None]]: + """Display labels by (target_type, target_id): a key's (alias, masked name), a + team's (alias, None), a user's (email, None).""" + return MappingProxyType( + { # mutable-ok: MappingProxyType needs a dict to wrap + key: value + for key, value in chain( + ((("key", row.token), (row.key_alias, row.key_name)) for row in key_rows), + ((("team", row.team_id), (row.team_alias, None)) for row in team_rows), + ((("user", row.user_id), (row.user_email, None)) for row in user_rows), + ) + } + ) + + +def _target_ids_of(responses: Sequence[ShadowEvalJobResponse], target_type: ShadowEvalTargetType) -> tuple[str, ...]: + return tuple( + sorted( + frozenset( + target.target_id + for response in responses + for target in response.targets + if target.target_type == target_type + ) + ) + ) + + +async def _with_target_labels( prisma_client: "PrismaClient", responses: Sequence[ShadowEvalJobResponse] ) -> tuple[ShadowEvalJobResponse, ...]: - """Resolve every scoped key's hash to its alias and masked name in one batched read, - so the UI can say whose traffic a job shadows. Deleted keys resolve to None.""" + """Resolve every scoped target's id to a display label in one batched read per kind, + so the UI can say whose traffic a job shadows: a key's alias and masked name, a + team's alias, a user's email. Deleted targets resolve to None.""" if not responses: return () - tokens: Final = sorted(frozenset(key.api_key_id for response in responses for key in response.keys)) - key_rows: Final = await _verification_tokens(prisma_client).find_many( - where={"token": {"in": tokens}} # mutable-ok: Prisma filter + tokens: Final = _target_ids_of(responses, "key") + team_ids: Final = _target_ids_of(responses, "team") + user_ids: Final = _target_ids_of(responses, "user") + key_rows: Final = ( + await _verification_tokens(prisma_client).find_many( + where={"token": {"in": list(tokens)}} # mutable-ok: Prisma filter + ) + if tokens + else () ) - labels: Final[Mapping[str, tuple[str | None, str | None]]] = { - row.token: (row.key_alias, row.key_name) for row in key_rows or () - } + team_rows: Final = ( + await _team_rows(prisma_client).find_many( + where={"team_id": {"in": list(team_ids)}} # mutable-ok: Prisma filter + ) + if team_ids + else () + ) + user_rows: Final = ( + await _user_rows(prisma_client).find_many( + where={"user_id": {"in": list(user_ids)}} # mutable-ok: Prisma filter + ) + if user_ids + else () + ) + labels: Final = _target_labels(key_rows or (), team_rows or (), user_rows or ()) return tuple( response.model_copy( update={ # mutable-ok: pydantic update payload - "keys": tuple( - key.model_copy( + "targets": tuple( + target.model_copy( update={ # mutable-ok: pydantic update payload - "key_alias": labels.get(key.api_key_id, _NO_KEY_LABELS)[0], - "key_name": labels.get(key.api_key_id, _NO_KEY_LABELS)[1], + "target_alias": labels.get((target.target_type, target.target_id), _NO_TARGET_LABELS)[0], + "key_name": labels.get((target.target_type, target.target_id), _NO_TARGET_LABELS)[1], } ) - for key in response.keys + for target in response.targets ) } ) @@ -1125,29 +1220,37 @@ async def _with_key_labels( ) -async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> ShadowEvalResult | None: - """All three stratifications of one job's verdicts. Tier answers "where does the router - do well"; the model stratification groups by whichever model served the real arm, so it - answers "which of the models these keys use today would the router beat" forward, and - "for the turns the router sent to X, did X beat the baseline" in reverse; key answers - "which key's traffic does the router suit". Reads are bounded by the job's own attempts - (<= the sum of its keys' max_turns) via the job_id index.""" +async def _shadow_eval_results( + prisma_client: "PrismaClient", legs: Sequence[_LegRow] +) -> tuple[ShadowEvalResult | None, Mapping[tuple[str, str], ShadowEvalSlice]]: + """One job's stratified verdicts, plus each target's own slice keyed by the + (target_type, target_id) pair so a key, team, and user sharing an id can never + collapse into one entry. Tier answers "where does the router do well"; the model + stratification groups by whichever model served the real arm, so it answers "which + of the models these targets use today would the router beat" forward, and "for the + turns the router sent to X, did X beat the baseline" in reverse; the per-target + slices answer "which target's traffic does the router suit". Reads are bounded by + the job's own attempts (<= the sum of its targets' max_turns) via the job_id index.""" leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param by_tier: Final = _ATTEMPT_AGG_ROWS.validate_python( await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, leg_ids) or () ) if not by_tier: - return None + return None, MappingProxyType({}) by_model: Final = _ATTEMPT_AGG_ROWS.validate_python( await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, leg_ids) or () ) - key_by_leg: Final = MappingProxyType({leg.id: leg.api_key_id for leg in legs}) + target_by_leg: Final = MappingProxyType({leg.id: (leg.target_type, leg.target_id) for leg in legs}) by_leg: Final = _ATTEMPT_AGG_ROWS.validate_python( await _query_raw(prisma_client, _ATTEMPT_AGG_BY_LEG_SQL, leg_ids) or () ) - by_key: Final = tuple( - row.model_copy(update={"grp": key_by_leg[row.grp]}) # mutable-ok: pydantic update payload - for row in by_leg + verdicts_by_target: Final[Mapping[tuple[str, str], ShadowEvalSlice]] = MappingProxyType( + { # mutable-ok: MappingProxyType needs a dict to wrap + target_by_leg[slice.group]: slice.model_copy( + update={"group": target_by_leg[slice.group][1]} # mutable-ok: pydantic update payload + ) + for slice in _slices(by_leg) + } ) total_turns: Final = sum(r.turn_count for r in by_tier) funnel_rows: Final = await _query_raw(prisma_client, _FUNNEL_TOTALS_SQL, leg_ids) @@ -1155,10 +1258,9 @@ async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_Le # Coverage only when EVERY leg has a funnel row: a partial seed (one leg's insert # failed) must read as unknown, not as job-level counts missing a leg's traffic. funnel: Final = counted if counted is not None and counted.legs_with_rows == len(leg_ids) else None - return ShadowEvalResult( + result: Final = ShadowEvalResult( by_tier=_slices(by_tier), by_current_model=_slices(by_model), - by_key=_slices(by_key), overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns), overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns), sampled_real_spend=sum(r.real_spend for r in by_tier), @@ -1168,6 +1270,7 @@ async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_Le shed_count=funnel.shed if funnel is not None else None, withheld_count=funnel.withheld if funnel is not None else None, ) + return result, verdicts_by_target @router.post( @@ -1182,22 +1285,29 @@ async def start_shadow_eval( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> ShadowEvalJobResponse: """ - Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against - a second arm, judge the two responses blind, and stratify win rates by tier, by the model - that served the real arm, and by key. + Start a shadow eval: duplicate a sampled slice of one or more targets' live traffic + against a second arm, judge the two responses blind, and stratify win rates by tier, + by the model that served the real arm, and by target. - A forward job answers whether the keys should adopt router_name: it samples the requests - the router did not serve and duplicates them through it. A reverse job answers whether a - key already on the router still gains from it: it samples the requests the router did - serve and duplicates them against baseline_model. A key can hold one active job per - direction, so both questions can run at once. + A target is a virtual key, a team, or a user. Team and user targets match on the + identity every request resolves to at auth time, so they cover JWT-authenticated + traffic, which presents no virtual key; a user target samples that user's traffic + across all their teams, whether it arrives on a JWT or a key they own. - Shadow responses are never served to users. Each key samples until its recorded eval - spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's - window ends, or the job is stopped, so one key running out of budget does not end - sampling for the others; sampling changes propagate to pods within about 10 seconds. - Shadow and judge calls bill to the shadowed key but are excluded from request counts - and auto-router adoption metrics. + A forward job answers whether the targets should adopt router_name: it samples the + requests the router did not serve and duplicates them through it. A reverse job + answers whether a target already on the router still gains from it: it samples the + requests the router did serve and duplicates them against baseline_model. A target + can hold one active job per direction, so both questions can run at once, and a + request matching several jobs' targets (say its key and its team) is sampled by + each, separately budgeted. + + Shadow responses are never served to users. Each target samples until its recorded + eval spend, the shadow and judge calls' own cost, reaches max_budget dollars, the + job's window ends, or the job is stopped, so one target running out of budget does + not end sampling for the others; sampling changes propagate to pods within about 10 + seconds. Shadow and judge calls bill to the sampled request's own identity but are + excluded from request counts and auto-router adoption metrics. """ from litellm.proxy.proxy_server import llm_router, prisma_client @@ -1206,35 +1316,88 @@ async def start_shadow_eval( raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) if llm_router is None or not _is_configured_pre_routing_strategy(llm_router, data.router_name): raise HTTPException(status_code=400, detail=f"'{data.router_name}' is not a configured auto-router") - token_rows: Final = await _verification_tokens(prisma_client).find_many( - where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter - ) - unknown: Final = tuple(sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ()))) - if unknown: - raise HTTPException( - status_code=400, - detail=( - f"api_key_ids not on this proxy: {', '.join(unknown)}; pass each key's token hash, " - "the value the key list and key info endpoints report" - ), + token_rows: Final = ( + await _verification_tokens(prisma_client).find_many( + where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter ) + if data.api_key_ids + else () + ) + team_rows: Final = ( + await _team_rows(prisma_client).find_many( + where={"team_id": {"in": list(data.team_ids)}} # mutable-ok: Prisma filter + ) + if data.team_ids + else () + ) + user_rows: Final = ( + await _user_rows(prisma_client).find_many( + where={"user_id": {"in": list(data.user_ids)}} # mutable-ok: Prisma filter + ) + if data.user_ids + else () + ) + unknown_keys: Final = sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ())) + unknown_teams: Final = sorted(frozenset(data.team_ids) - frozenset(row.team_id for row in team_rows or ())) + unknown_users: Final = sorted(frozenset(data.user_ids) - frozenset(row.user_id for row in user_rows or ())) + unknown_parts: Final = tuple( + part + for part in ( + ( + f"api_key_ids not on this proxy: {', '.join(unknown_keys)}; pass each key's token hash, " + "the value the key list and key info endpoints report" + ) + if unknown_keys + else None, + f"team_ids not on this proxy: {', '.join(unknown_teams)}" if unknown_teams else None, + f"user_ids not on this proxy: {', '.join(unknown_users)}" if unknown_users else None, + ) + if part is not None + ) + if unknown_parts: + raise HTTPException(status_code=400, detail=". ".join(unknown_parts)) # Every model check below runs once per team the job samples for, since that is the # identity the shadow and judge calls carry and therefore what the router selects on. - team_ids: Final = tuple(dict.fromkeys(row.team_id for row in token_rows or ())) + # A user target's traffic can span teams, so it validates unscoped (None); each + # sampled attempt still resolves the judge under its own request's team at eval time. + team_ids: Final = tuple( + dict.fromkeys( + ( + *(row.team_id for row in token_rows or ()), + *data.team_ids, + *((None,) if data.user_ids else ()), + ) + ) + ) _validate_plain_model(llm_router, data.judge_model, "judge_model", team_ids) if data.baseline_model is not None: _validate_plain_model(llm_router, data.baseline_model, "baseline_model", team_ids) _validate_judge_is_not_a_candidate(llm_router, data, team_ids) + requested_targets: Final[tuple[tuple[ShadowEvalTargetType, str], ...]] = ( + *(("key", key) for key in data.api_key_ids), + *(("team", team) for team in data.team_ids), + *(("user", user) for user in data.user_ids), + ) + requested_by_type: Final[tuple[tuple[ShadowEvalTargetType, tuple[str, ...]], ...]] = tuple( + (target_type, ids) + for target_type, ids in (("key", data.api_key_ids), ("team", data.team_ids), ("user", data.user_ids)) + if ids + ) # A job whose window passed or whose budget ran out stopped sampling on its own, - # but its legs still hold their slots in the per-key, per-direction partial unique index - # until stamped; free them so a new eval can start. Sweeping both directions is deliberate. - requested: Final = list(data.api_key_ids) # mutable-ok: query param - await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, requested) + # but its legs still hold their slots in the per-target, per-direction partial unique + # index until stamped; free them so a new eval can start. Sweeping both directions is + # deliberate. Sweep and claim filter on exact (target_type, id) pairs so a team id + # that happens to equal a key hash never matches the other kind's slot. + for target_type, ids in requested_by_type: + await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, list(ids), target_type) # mutable-ok: query param claimed: Final = await _shadow_eval_jobs(prisma_client).find_many( where={ # mutable-ok: Prisma filter - "api_key_id": {"in": requested}, # mutable-ok: Prisma filter + "OR": [ # mutable-ok: Prisma filter + {"target_type": target_type, "target_id": {"in": list(ids)}} # mutable-ok: Prisma filter + for target_type, ids in requested_by_type + ], "direction": data.direction, "stopped_at": None, }, @@ -1244,7 +1407,7 @@ async def start_shadow_eval( status_code=409, detail=( f"Already in an active {data.direction} shadow eval job: " - + ", ".join(sorted(f"{row.api_key_id} (job {row.group_id})" for row in claimed)) + + ", ".join(sorted(f"{row.target_type} {row.target_id} (job {row.group_id})" for row in claimed)) + ". Stop it first." ), ) @@ -1268,10 +1431,16 @@ async def start_shadow_eval( # Leg ids are minted here rather than by the DB default so the funnel seed below # writes from the same values with no read-back, which a lagging read replica # (DATABASE_URL_READ_REPLICA) could otherwise return empty. - leg_ids: Final = tuple(str(uuid4()) for _ in data.api_key_ids) + leg_ids: Final = tuple(str(uuid4()) for _ in requested_targets) await _shadow_eval_jobs(prisma_client).create_many( data=[ # mutable-ok: Prisma payload - {**shared_config, "id": leg_id, "api_key_id": key} for leg_id, key in zip(leg_ids, data.api_key_ids) + { # mutable-ok: Prisma payload + **shared_config, + "id": leg_id, + "target_type": target_type, + "target_id": target_id, + } # mutable-ok: Prisma payload + for leg_id, (target_type, target_id) in zip(leg_ids, requested_targets) ] ) except Exception as e: @@ -1280,7 +1449,8 @@ async def start_shadow_eval( raise HTTPException( status_code=409, detail=( - f"A requested key was claimed by another {data.direction} shadow eval job concurrently. Stop it first." + f"A requested target was claimed by another {data.direction} shadow eval job concurrently. " + "Stop it first." ), ) from e # Seed a zero funnel row per leg NOW: a fully covered job never skips a request, so @@ -1293,18 +1463,19 @@ async def start_shadow_eval( ) except Exception as seed_err: # noqa: BLE001 # coverage is advisory; the job must still start verbose_proxy_logger.error("shadow_eval: funnel seed failed for job %s: %s", group_id, seed_err) - labels: Final = MappingProxyType({row.token: row for row in token_rows}) + labels: Final = _target_labels(token_rows or (), team_rows or (), user_rows or ()) return ShadowEvalJobResponse( job_id=group_id, - keys=tuple( - ShadowEvalJobKeyResponse( - api_key_id=api_key_id, + targets=tuple( + ShadowEvalJobTargetResponse( + target_type=target_type, + target_id=target_id, max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=data.max_budget, - key_alias=labels[api_key_id].key_alias, - key_name=labels[api_key_id].key_name, + target_alias=labels.get((target_type, target_id), _NO_TARGET_LABELS)[0], + key_name=labels.get((target_type, target_id), _NO_TARGET_LABELS)[1], ) - for api_key_id in sorted(data.api_key_ids) + for target_type, target_id in sorted(requested_targets) ), router_name=data.router_name, direction=data.direction, @@ -1324,22 +1495,29 @@ async def start_shadow_eval( ) async def list_shadow_eval_jobs( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], - api_key_id: Annotated[ - str | None, Query(description="Filter to jobs that shadow this key, alone or alongside others") + target_type: Annotated[ + ShadowEvalTargetType | None, Query(description="Kind of target to filter on; requires target_id") + ] = None, + target_id: Annotated[ + str | None, Query(description="Filter to jobs that shadow this target, alone or alongside others") ] = None, limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50, ) -> tuple[ShadowEvalJobResponse, ...]: - """List shadow eval jobs, newest first, each key with its attempt count so status is - accurate. Judged counts, spend, and results ride the detail endpoint only.""" + """List shadow eval jobs, newest first, each target with its attempt count so status + is accurate. Judged counts, spend, and results ride the detail endpoint only.""" from litellm.proxy.proxy_server import prisma_client _require_admin_viewer(user_api_key_dict, "view shadow evals") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + filter_type: Final = target_type if isinstance(target_type, str) else None + filter_id: Final = target_id if isinstance(target_id, str) else None + if (filter_type is None) != (filter_id is None): + raise HTTPException(status_code=400, detail="target_type and target_id filter together; pass both or neither") legs: Final = _LEG_ROWS.validate_python( ( - await _query_raw(prisma_client, _LIST_LEGS_BY_KEY_SQL, limit, api_key_id) - if api_key_id + await _query_raw(prisma_client, _LIST_LEGS_BY_TARGET_SQL, limit, filter_type, filter_id) + if filter_type and filter_id else await _query_raw(prisma_client, _LIST_LEGS_SQL, limit) ) or () @@ -1354,7 +1532,7 @@ async def list_shadow_eval_jobs( by_group, key=lambda group_id: max(leg.created_at for leg in by_group[group_id]), reverse=True ) counts: Final = await _leg_attempt_counts(prisma_client, legs) - return await _with_key_labels( + return await _with_target_labels( prisma_client, tuple(_group_response(group_id, by_group[group_id], counts) for group_id in newest_first) ) @@ -1391,16 +1569,25 @@ async def get_shadow_eval_job( where={"job_id": {"in": leg_ids}, "outcome": "error"}, # mutable-ok: Prisma filter order={"created_at": "desc"}, # mutable-ok: Prisma order ) - labeled: Final = await _with_key_labels( + labeled: Final = await _with_target_labels( prisma_client, (_group_response(job_id, legs, await _leg_attempt_counts(prisma_client, legs)),) ) + results, verdicts_by_target = await _shadow_eval_results(prisma_client, legs) return labeled[0].model_copy( update={ # mutable-ok: pydantic update payload "judged_count": totals[0].judged_count if totals else 0, "error_count": totals[0].error_count if totals else 0, "judge_spend": round(totals[0].judge_spend, 6) if totals else 0.0, "last_error": latest_error.error if latest_error else None, - "results": await _shadow_eval_results(prisma_client, legs), + "results": results, + "targets": tuple( + target.model_copy( + update={ # mutable-ok: pydantic update payload + "verdicts": verdicts_by_target.get((target.target_type, target.target_id)) + } + ) + for target in labeled[0].targets + ), } ) @@ -1415,8 +1602,8 @@ async def stop_shadow_eval_job( job_id: str, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> ShadowEvalJobResponse: - """Stop an active shadow eval job, every key it scopes at once. Attempts are kept; - sampling halts within ~10s. Keys that already stopped on their own budget keep the + """Stop an active shadow eval job, every target it scopes at once. Attempts are kept; + sampling halts within ~10s. Targets that already stopped on their own budget keep the stopped_at they earned. The statement is the whole state machine: it claims the job only while a leg still samples inside the window with no stop recorded, so a racing operator, a same-instant budget spend, and a repeat stop all read the same 400 with @@ -1443,5 +1630,5 @@ async def stop_shadow_eval_job( current: Final = _group_response(job_id, legs, counts) if claimed == 0: raise HTTPException(status_code=400, detail=f"Job {job_id} is already {current.status}") - labeled: Final = await _with_key_labels(prisma_client, (current,)) + labeled: Final = await _with_target_labels(prisma_client, (current,)) return labeled[0] diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 60223265211..01a607b68a9 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession { model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) group_id String // legs of one job share this; the API's job id - api_key_id String // hashed virtual key whose traffic this leg shadows + target_type String @default("key") // key | team | user + target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise - max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets + max_budget Float? // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets created_at DateTime @default(now()) created_by String? ends_at DateTime @@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob { stopped_by String? // operator who stopped it early; null when it ended on its own @@index([group_id]) - @@index([api_key_id]) + @@index([target_type, target_id]) @@index([created_at]) } diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index bde3f5f9e7e..dfc45ccf9bf 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -245,6 +245,8 @@ ShadowEvalStatus: TypeAlias = Literal["running", "completed", "stopped"] ShadowEvalDirection: TypeAlias = Literal["forward", "reverse"] +ShadowEvalTargetType: TypeAlias = Literal["key", "team", "user"] + DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5" # Sample-count ceiling written on every new job: a zero-cost error loop (a shadow arm that @@ -253,16 +255,37 @@ SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000 class StartShadowEvalRequest(BaseModel): - """Start duplicating one or more keys' traffic for blind comparison against an auto-router.""" + """Start duplicating one or more targets' traffic for blind comparison against an auto-router. + + A target is a virtual key, a team, or a user; each becomes its own leg with its own + budget and stop state. Team and user targets match on the identity every request + carries after auth (user_api_key_team_id / user_api_key_user_id), so they cover + JWT-authenticated traffic, which presents no virtual key at all.""" api_key_ids: tuple[str, ...] = Field( - min_length=1, + default=(), max_length=100, description=( - "The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these " - "keys' traffic; requests made with any other key are not sampled. Each key carries its own " - "max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 " - "keys per job, which also bounds every read the job's endpoints make." + "Hashed virtual keys whose traffic will be shadowed. Combined with team_ids and user_ids the job " + "needs at least one target and at most 100, which also bounds every read the job's endpoints make. " + "Each target carries its own max_budget spend budget, so one exhausting its budget leaves the " + "others sampling." + ), + ) + team_ids: tuple[str, ...] = Field( + default=(), + max_length=100, + description=( + "Teams whose traffic will be shadowed, matched on the team every authenticated request resolves " + "to, so a team's JWT-auth and virtual-key traffic are both sampled" + ), + ) + user_ids: tuple[str, ...] = Field( + default=(), + max_length=100, + description=( + "Users whose traffic will be shadowed, matched on the user every authenticated request resolves " + "to across all their teams: JWT requests carrying their subject claim and virtual keys they own" ), ) router_name: str = Field(description="The auto-router under evaluation, in either direction") @@ -285,7 +308,7 @@ class StartShadowEvalRequest(BaseModel): shadow_percentage: float = Field( ge=0.1, le=100.0, - description="Percentage of the key's requests to duplicate through the router", + description="Percentage of each target's requests to duplicate through the router", ) judge_model: str = Field( default=DEFAULT_SHADOW_EVAL_JUDGE_MODEL, @@ -306,9 +329,9 @@ class StartShadowEvalRequest(BaseModel): ge=0.01, le=10_000, description=( - "Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with " - "the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval " - "spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight " + "Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with " + "the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval " + "spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight " "samples can overshoot the cap by one sampling cache window" ), ) @@ -319,7 +342,7 @@ class StartShadowEvalRequest(BaseModel): """Pydantic ignores unknown fields, so a caller still sending max_turns would silently run on the default dollar budget instead of the bound they asked for.""" if isinstance(values, Mapping) and "max_turns" in values: - raise ValueError("max_turns was replaced by max_budget, the per-key USD cap on the eval's own spend") + raise ValueError("max_turns was replaced by max_budget, the per-target USD cap on the eval's own spend") return values @field_validator("shadow_percentage") @@ -327,12 +350,21 @@ class StartShadowEvalRequest(BaseModel): def _round_percentage(cls, value: float) -> float: return round(value, 2) - @field_validator("api_key_ids") + @field_validator("api_key_ids", "team_ids", "user_ids") @classmethod - def _dedupe_keys(cls, value: tuple[str, ...]) -> tuple[str, ...]: - """A key named twice would collide with itself on the one-active-per-(key, direction) index.""" + def _dedupe_targets(cls, value: tuple[str, ...]) -> tuple[str, ...]: + """A target named twice would collide with itself on the one-active-per-(target, direction) index.""" return tuple(dict.fromkeys(value)) + @model_validator(mode="after") + def _at_least_one_target_at_most_hundred(self) -> "StartShadowEvalRequest": + total: Final = len(self.api_key_ids) + len(self.team_ids) + len(self.user_ids) + if total < 1: + raise ValueError("at least one target is required: pass api_key_ids, team_ids, or user_ids") + if total > 100: + raise ValueError("at most 100 targets per job across api_key_ids, team_ids, and user_ids") + return self + @model_validator(mode="after") def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest": if self.direction == "reverse" and self.baseline_model is None: @@ -343,8 +375,9 @@ class StartShadowEvalRequest(BaseModel): class ShadowEvalSlice(BaseModel): - """Judge outcomes for one slice of a job's verdicts (a router tier, or one of the - models that served the real arm).""" + """Judge outcomes for one slice of a job's verdicts: a router tier, one of the + models that served the real arm, or one scoped target (embedded on that target's + own entry, so slices never need re-joining to a target by id).""" group: str turn_count: int @@ -395,12 +428,6 @@ class ShadowEvalResult(BaseModel): "and in reverse the models the router itself picked" ) ) - by_key: tuple[ShadowEvalSlice, ...] = Field( - description=( - "One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job " - "scopes but has not judged a turn for yet are absent rather than reported as zero" - ), - ) overall_shadow_win_rate_pct: float overall_tie_rate_pct: float sampled_real_spend: float = Field( @@ -436,27 +463,28 @@ class ShadowEvalResult(BaseModel): ) -class ShadowEvalJobKeyResponse(BaseModel): - """One key a job shadows, with its own budget and stop state.""" +class ShadowEvalJobTargetResponse(BaseModel): + """One target a job shadows (a key, team, or user), with its own budget and stop state.""" - api_key_id: str = Field(description="The hashed virtual key whose traffic this entry scopes") + target_type: ShadowEvalTargetType = Field(description="What kind of entity this entry scopes") + target_id: str = Field(description="The hashed virtual key, team id, or user id whose traffic this entry scopes") max_turns: int = Field( description=( - "This key's sample-count ceiling: the whole budget for jobs created before max_budget " + "This target's sample-count ceiling: the whole budget for jobs created before max_budget " "existed, and the error-loop safety valve otherwise" ) ) max_budget: float | None = Field( default=None, description=( - "This key's own USD budget for the eval's shadow and judge spend, independent of its " + "This target's own USD budget for the eval's shadow and judge spend, independent of its " "siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds" ), ) stopped_at: datetime | None = Field( default=None, description=( - "When this key's slot was stamped free, whether its own budget ran out, the window closed, " + "When this target's slot was stamped free, whether its own budget ran out, the window closed, " "or an operator stopped the job; status is derived, so a spent budget reads completed even " "while this is still unset" ), @@ -464,45 +492,53 @@ class ShadowEvalJobKeyResponse(BaseModel): attempt_count: int | None = Field( default=None, description=( - "This key's sampled attempts so far, judged and errored alike, the same count the sampler " + "This target's sampled attempts so far, judged and errored alike, the same count the sampler " "budgets against max_turns; populated on list and detail responses. Frozen at stopped_at " - "once the key is stamped, so in-flight attempts landing after a stop never reclassify it" + "once the target is stamped, so in-flight attempts landing after a stop never reclassify it" ), ) spend: float | None = Field( default=None, description=( - "This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets " + "This target's recorded shadow plus judge spend in USD, the same figure the sampler budgets " "against max_budget; populated on list and detail responses and frozen at stopped_at " "exactly like attempt_count" ), ) + verdicts: "ShadowEvalSlice | None" = Field( + default=None, + description="This target's own judged-verdict slice; detail endpoint only, None until a turn is judged", + ) + @property def budget_spent(self) -> bool: over_spend: Final = self.max_budget is not None and self.spend is not None and self.spend >= self.max_budget return over_spend or (self.attempt_count is not None and self.attempt_count >= self.max_turns) - key_alias: str | None = Field( + target_alias: str | None = Field( default=None, - description="Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted", + description=( + "Display label resolved from the target's own row at read time: the key's alias, the team's " + "alias, or the user's email; None when unset or deleted" + ), ) key_name: str | None = Field( default=None, - description="Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias", + description="Masked display name (sk-...) for key targets, resolved at read time; None for teams and users", ) class ShadowEvalJobResponse(BaseModel): - """A shadow-eval job over one or more keys, each with its own budget and stop state; - status is derived from stopped_by, the keys' stop and budget state, and ends_at, + """A shadow-eval job over one or more targets, each with its own budget and stop state; + status is derived from stopped_by, the targets' stop and budget state, and ends_at, never stored, so no writer anywhere can produce an inconsistent one. Aggregate fields are populated by the detail endpoint only and stay None on list responses.""" job_id: str - keys: tuple[ShadowEvalJobKeyResponse, ...] = Field( + targets: tuple[ShadowEvalJobTargetResponse, ...] = Field( min_length=1, - description="The keys whose traffic this job evaluates, and only those keys', each with its own budget", + description="The targets whose traffic this job evaluates, and only theirs, each with its own budget", ) router_name: str direction: ShadowEvalDirection = "forward" @@ -531,8 +567,8 @@ class ShadowEvalJobResponse(BaseModel): def status(self) -> ShadowEvalStatus: """Three recorded facts, no history-guessing: a stop is stopped_by (the migration backfills it for every job that displayed stopped when the column arrived, so the - pre-column population is closed), completion is the window passing or every key - spending its budget, and anything else is running. The all-keys-stamped fallback + pre-column population is closed), completion is the window passing or every target + spending its budget, and anything else is running. The all-targets-stamped fallback covers only stops written by pre-column pods during a rolling deploy.""" if self.stopped_by is not None: return "stopped" @@ -540,8 +576,8 @@ class ShadowEvalJobResponse(BaseModel): self.ends_at if self.ends_at.tzinfo else self.ends_at.replace(tzinfo=timezone.utc) ): return "completed" - if all(key.budget_spent for key in self.keys): + if all(target.budget_spent for target in self.targets): return "completed" - if all(key.stopped_at is not None for key in self.keys): + if all(target.stopped_at is not None for target in self.targets): return "stopped" return "running" diff --git a/schema.prisma b/schema.prisma index 60223265211..01a607b68a9 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1529,14 +1529,15 @@ model LiteLLM_AutoRouterSession { model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) group_id String // legs of one job share this; the API's job id - api_key_id String // hashed virtual key whose traffic this leg shadows + target_type String @default("key") // key | team | user + target_id String // hashed virtual key, team_id, or user_id whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise - max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets + max_budget Float? // per-target USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets created_at DateTime @default(now()) created_by String? ends_at DateTime @@ -1544,7 +1545,7 @@ model LiteLLM_ShadowEvalJob { stopped_by String? // operator who stopped it early; null when it ended on its own @@index([group_id]) - @@index([api_key_id]) + @@index([target_type, target_id]) @@index([created_at]) } diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index f9c287fc7b7..1af3dd3f613 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -58,11 +58,12 @@ def _prisma(jobs=(), attempt_counts=(), attempt_costs=()) -> MagicMock: return prisma -def _job_record(job: ActiveShadowEvalJob, api_key_id="key-hash") -> MagicMock: +def _job_record(job: ActiveShadowEvalJob, target_type="key", target_id="key-hash") -> MagicMock: record = MagicMock() for field, value in dict( id=job.id, - api_key_id=api_key_id, + target_type=target_type, + target_id=target_id, router_name=job.router_name, direction=job.direction, baseline_model=job.baseline_model, @@ -123,7 +124,7 @@ def _spend_counter(store=None): return counter, read, write -def _logger(router=None, prisma=None, jobs=(), counter_store=None) -> ShadowEvalLogger: +def _logger(router=None, prisma=None, jobs=(), counter_store=None, jobs_by_target=None) -> ShadowEvalLogger: cache = InMemoryCache(max_size_in_memory=4, default_ttl=60) counter, read, write = _spend_counter(counter_store) funnel_events = [] @@ -137,8 +138,9 @@ def _logger(router=None, prisma=None, jobs=(), counter_store=None) -> ShadowEval ) logger._test_counter = counter logger._test_funnel = funnel_events - if jobs: - cache.set_cache("shadow_eval:active_jobs", {"key-hash": tuple(jobs)}) + seeded = jobs_by_target if jobs_by_target is not None else ({("key", "key-hash"): tuple(jobs)} if jobs else None) + if seeded is not None: + cache.set_cache("shadow_eval:active_jobs", seeded) return logger @@ -837,6 +839,86 @@ class TestSuccessHookSkipChain: prisma.db.litellm_shadowevalattempt.create.assert_not_called() +JWT_IDENTITY = {"user_api_key_hash": None, "user_api_key_team_id": "team-eng", "user_api_key_user_id": "dev-alice"} + + +@pytest.mark.asyncio +class TestTargetMatching: + """A request qualifies for a job through ANY of its resolved identities: key hash, + team id, or user id. Team and user jobs must therefore sample JWT-authenticated + traffic, which carries no key hash at all.""" + + @pytest.mark.parametrize( + "target,sampled", + [ + (("team", "team-eng"), True), + (("user", "dev-alice"), True), + (("key", "some-key"), False), + ], + ids=["team-job-samples-jwt-traffic", "user-job-samples-jwt-traffic", "key-jobs-never-match-keyless-traffic"], + ) + async def test_jwt_shaped_traffic_matches_team_and_user_jobs_but_no_key_job(self, target, sampled): + prisma = _prisma() + router = _router() + logger = _logger(router=router, prisma=prisma, jobs_by_target={target: (_job(),)}) + hook_kwargs = _success_kwargs() + hook_kwargs["standard_logging_object"]["metadata"] = dict(JWT_IDENTITY) + + await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None) + await _drain(logger) + + if sampled: + prisma.db.litellm_shadowevalattempt.create.assert_awaited_once() + assert prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]["job_id"] == "job-1" + else: + router.acompletion.assert_not_called() + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + + async def test_an_event_with_no_identity_early_returns_without_a_cache_read(self): + prisma = _prisma() + router = _router() + cache = MagicMock(spec=InMemoryCache) + cache.async_get_cache = AsyncMock() + logger = ShadowEvalLogger( + router_provider=lambda: router, + prisma_provider=lambda: prisma, + jobs_cache=cache, + ) + hook_kwargs = _success_kwargs() + hook_kwargs["standard_logging_object"]["metadata"] = {} + + await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None) + + cache.async_get_cache.assert_not_awaited() + router.acompletion.assert_not_called() + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + + async def test_an_event_matching_a_key_job_and_a_team_job_fires_both(self): + """A request's key and its team can each hold a job; the two are separately + budgeted experiments, so both fire and each counts its own start.""" + prisma = _prisma() + logger = _logger( + router=_router(), + prisma=prisma, + jobs_by_target={ + ("key", "key-hash"): (_job(id="key-job"),), + ("team", "team-eng"): (_job(id="team-job"),), + }, + ) + hook_kwargs = _success_kwargs() + hook_kwargs["standard_logging_object"]["metadata"] = { + "user_api_key_hash": "key-hash", + "user_api_key_team_id": "team-eng", + } + + await logger.async_log_success_event(hook_kwargs, RESPONSE, None, None) + await _drain(logger) + + rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.call_args_list] + assert sorted(row["job_id"] for row in rows) == ["key-job", "team-job"] + assert logger._job_starts == {"key-job": 1, "team-job": 1} + + @pytest.mark.asyncio class TestActiveJobsCache: async def test_cache_miss_reads_db_once_then_serves_from_cache(self): @@ -851,8 +933,8 @@ class TestActiveJobsCache: first = await logger._active_jobs() second = await logger._active_jobs() - assert [job.id for job in first["key-hash"]] == ["job-1"] - assert second["key-hash"][0].attempts == 7 + assert [job.id for job in first[("key", "key-hash")]] == ["job-1"] + assert second[("key", "key-hash")][0].attempts == 7 assert prisma.db.litellm_shadowevaljob.find_many.await_count == 1 where = prisma.db.litellm_shadowevaljob.find_many.call_args.kwargs["where"] assert where["stopped_at"] is None @@ -899,8 +981,8 @@ class TestActiveJobsCache: jobs = await logger._active_jobs() assert logger._job_starts == {} - assert jobs["key-hash"][0].attempts == 7 - assert jobs["key-hash"][0].spend == 0.05 + assert jobs[("key", "key-hash")][0].attempts == 7 + assert jobs[("key", "key-hash")][0].spend == 0.05 @pytest.mark.asyncio @@ -1249,13 +1331,14 @@ class TestActiveJobsFailClosed: jobs_cache=InMemoryCache(max_size_in_memory=4, default_ttl=60), ) - assert [job.id for job in (await logger._active_jobs())["key-hash"]] == ["job-ok"] + assert [job.id for job in (await logger._active_jobs())[("key", "key-hash")]] == ["job-ok"] - async def test_both_of_a_key_s_jobs_survive_the_lookup(self): + async def test_every_targets_jobs_survive_the_lookup_keyed_by_type_and_id(self): records = [ _job_record(_job(id="job-forward")), _job_record(_reverse_job(id="job-reverse")), - _job_record(_job(id="job-other"), api_key_id="other-key"), + _job_record(_job(id="job-other"), target_id="other-key"), + _job_record(_job(id="job-team"), target_type="team", target_id="team-eng"), ] prisma = _prisma(jobs=records, attempt_counts=[("job-reverse", 3)]) logger = ShadowEvalLogger( @@ -1266,9 +1349,11 @@ class TestActiveJobsFailClosed: jobs = await logger._active_jobs() - assert sorted(job.id for job in jobs["key-hash"]) == ["job-forward", "job-reverse"] - assert [job.id for job in jobs["other-key"]] == ["job-other"] - assert {job.id: job.attempts for job in jobs["key-hash"]}["job-reverse"] == 3 + assert sorted(job.id for job in jobs[("key", "key-hash")]) == ["job-forward", "job-reverse"] + assert [job.id for job in jobs[("key", "other-key")]] == ["job-other"] + assert [job.id for job in jobs[("team", "team-eng")]] == ["job-team"] + assert ("team-eng",) not in jobs and "team-eng" not in jobs + assert {job.id: job.attempts for job in jobs[("key", "key-hash")]}["job-reverse"] == 3 def _failing_router(): diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 726e09f3162..a74aa553449 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -878,7 +878,8 @@ def _leg_record(**overrides: object) -> MagicMock: defaults = { "id": "leg-1", "group_id": "job-1", - "api_key_id": "key-hash", + "target_type": "key", + "target_id": "key-hash", "router_name": "my-router", "direction": "forward", "baseline_model": None, @@ -912,8 +913,28 @@ def _key_record( return record +def _team_record(team_id: str, team_alias: str | None) -> MagicMock: + record = MagicMock(spec=["team_id", "team_alias"]) + record.team_id = team_id + record.team_alias = team_alias + return record + + +def _user_record(user_id: str, user_email: str | None) -> MagicMock: + record = MagicMock(spec=["user_id", "user_email"]) + record.user_id = user_id + record.user_email = user_email + return record + + def _shadow_prisma( - legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-hash", "key-hash-2"), key_teams=None + legs=(), + agg_rows=None, + by_leg_rows=None, + known_keys=("key-hash", "key-hash-2"), + key_teams=None, + known_teams=None, + known_users=None, ) -> MagicMock: """The job-table fake honours the filters it is handed, so a read that forgets stopped_at sees rows the partial index would have released, one that forgets @@ -921,6 +942,8 @@ def _shadow_prisma( group read that matched on a leg id would come back empty.""" prisma = MagicMock() teams: Final = key_teams or {} + team_aliases: Final = known_teams or {} + user_emails: Final = known_users or {} async def find_tokens(*, where): """Honours the token filter, like the job-table fake below: the endpoint derives the @@ -931,6 +954,17 @@ def _shadow_prisma( prisma.db.litellm_verificationtoken.find_many = AsyncMock(side_effect=find_tokens) + async def find_teams(*, where): + requested = where["team_id"]["in"] + return [_team_record(t, alias) for t, alias in team_aliases.items() if t in requested] + + async def find_users(*, where): + requested = where["user_id"]["in"] + return [_user_record(u, email) for u, email in user_emails.items() if u in requested] + + prisma.db.litellm_teamtable.find_many = AsyncMock(side_effect=find_teams) + prisma.db.litellm_usertable.find_many = AsyncMock(side_effect=find_users) + async def execute_raw(sql: str, *params: object): if "SET stopped_by" in sql: group = [row for row in stored if row.group_id == params[0]] @@ -959,9 +993,19 @@ def _shadow_prisma( async def find_many_legs(where=None, **_: object): current = list(stored) w = dict(where or {}) - if "api_key_id" in w: - wanted = w["api_key_id"]["in"] if isinstance(w["api_key_id"], dict) else [w["api_key_id"]] - current = [row for row in current if row.api_key_id in wanted] + if "OR" in w: + pairs = [ + ( + branch["target_type"], + branch["target_id"]["in"] if isinstance(branch["target_id"], dict) else [branch["target_id"]], + ) + for branch in w["OR"] + ] + current = [ + row + for row in current + if any(row.target_type == target_type and row.target_id in ids for target_type, ids in pairs) + ] if "direction" in w: current = [row for row in current if row.direction == w["direction"]] if "stopped_at" in w: @@ -983,7 +1027,8 @@ def _shadow_prisma( fields = ( "id", "group_id", - "api_key_id", + "target_type", + "target_id", "router_name", "direction", "baseline_model", @@ -1009,7 +1054,11 @@ def _shadow_prisma( if "AS attempt_count" in sql: return prisma.attempt_rows if "GROUP BY group_id" in sql: - scoped = [row for row in stored if "api_key_id = $2" not in sql or row.api_key_id == params[1]] + scoped = [ + row + for row in stored + if "target_type = $2" not in sql or (row.target_type == params[1] and row.target_id == params[2]) + ] keep = set(newest_groups(scoped, params[0])) return [leg_dict(row) for row in stored if row.group_id in keep] if "FILTER (WHERE outcome != 'error')::int AS judged_count" in sql: @@ -1058,7 +1107,7 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp response = await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) - sweep_sql, sweep_keys = prisma.db.execute_raw.call_args.args + sweep_sql, sweep_ids, sweep_type = prisma.db.execute_raw.call_args.args assert "stopped_at IS NULL" in sweep_sql assert "j.ends_at <= (NOW() AT TIME ZONE 'utc')" in sweep_sql assert "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in sweep_sql @@ -1066,12 +1115,13 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp assert "j.max_budget IS NOT NULL" in sweep_sql assert ">= j.max_budget" in sweep_sql assert "SUM(a.judge_cost + a.shadow_cost + a.shadow_classifier_cost)" in sweep_sql - assert "j.api_key_id = ANY($1::text[])" in sweep_sql - assert sweep_keys == ["key-hash", "key-hash-2"] + assert "j.target_type = $2 AND j.target_id = ANY($1::text[])" in sweep_sql + assert sweep_ids == ["key-hash", "key-hash-2"] + assert sweep_type == "key" prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] - assert [row["api_key_id"] for row in rows] == ["key-hash", "key-hash-2"] - assert len({frozenset((k, v) for k, v in row.items() if k not in ("api_key_id", "id")) for row in rows}) == 1 + assert [(row["target_type"], row["target_id"]) for row in rows] == [("key", "key-hash"), ("key", "key-hash-2")] + assert len({frozenset((k, v) for k, v in row.items() if k not in ("target_id", "id")) for row in rows}) == 1 assert len({row["id"] for row in rows}) == len(rows) assert len({row["group_id"] for row in rows}) == 1 assert all(row["max_turns"] == SHADOW_EVAL_TURN_VALVE and row["created_by"] == "admin" for row in rows) @@ -1080,11 +1130,12 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp assert response.job_id == rows[0]["group_id"] assert response.status == "running" assert response.judged_count is None - assert [(key.api_key_id, key.max_budget, key.key_alias) for key in response.keys] == [ + assert [(target.target_id, target.max_budget, target.target_alias) for target in response.targets] == [ ("key-hash", 5.0, "prod-alpha"), ("key-hash-2", 5.0, "prod-alpha"), ] - assert all(key.max_turns == SHADOW_EVAL_TURN_VALVE for key in response.keys) + assert all(target.target_type == "key" for target in response.targets) + assert all(target.max_turns == SHADOW_EVAL_TURN_VALVE for target in response.targets) @pytest.mark.asyncio @@ -1232,7 +1283,7 @@ async def test_start_shadow_eval_rejections( import litellm.proxy.proxy_server as proxy_server _configure_anthropic_sdk_judge(monkeypatch) - prisma = _shadow_prisma(legs=[_leg_record(id=f"leg-{key}", group_id="job-7", api_key_id=key) for key in claimed]) + prisma = _shadow_prisma(legs=[_leg_record(id=f"leg-{key}", group_id="job-7", target_id=key) for key in claimed]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) @@ -1315,14 +1366,14 @@ async def test_start_shadow_eval_names_the_busy_key_and_its_job(monkeypatch: pyt import litellm.proxy.proxy_server as proxy_server _configure_anthropic_sdk_judge(monkeypatch) - prisma = _shadow_prisma(legs=[_leg_record(id="leg-b", group_id="job-7", api_key_id="key-hash-2")]) + prisma = _shadow_prisma(legs=[_leg_record(id="leg-b", group_id="job-7", target_id="key-hash-2")]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) with pytest.raises(HTTPException) as exc: await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) assert exc.value.status_code == 409 - assert "key-hash-2 (job job-7)" in exc.value.detail + assert "key key-hash-2 (job job-7)" in exc.value.detail @pytest.mark.asyncio @@ -1441,6 +1492,180 @@ def test_start_shadow_eval_request_dedupes_and_bounds_the_key_set(): _start_request(api_key_ids=tuple(f"k{i}" for i in range(101))) +def test_start_request_bounds_the_combined_target_count_across_types(): + """The 1..100 bound counts keys, teams, and users together, so a caller cannot dodge + it by spreading targets over the three fields, and a request naming no target of any + type samples nothing and is rejected.""" + with pytest.raises(ValidationError, match="at least one target"): + _start_request(api_key_ids=(), team_ids=(), user_ids=()) + with pytest.raises(ValidationError, match="at most 100 targets"): + _start_request(api_key_ids=tuple(f"k{i}" for i in range(60)), team_ids=tuple(f"t{i}" for i in range(41))) + mixed = _start_request(api_key_ids=tuple(f"k{i}" for i in range(60)), team_ids=tuple(f"t{i}" for i in range(40))) + assert len(mixed.api_key_ids) + len(mixed.team_ids) == 100 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "overrides,prisma_kwargs,expected_target", + [ + ( + {"api_key_ids": (), "team_ids": ("team-eng",)}, + {"known_teams": {"team-eng": "Engineering"}}, + ("team", "team-eng", "Engineering"), + ), + ( + {"api_key_ids": (), "user_ids": ("dev-alice",)}, + {"known_users": {"dev-alice": "alice@example.com"}}, + ("user", "dev-alice", "alice@example.com"), + ), + ], + ids=["team-target-labeled-by-team-alias", "user-target-labeled-by-user-email"], +) +async def test_start_shadow_eval_creates_typed_legs_for_team_and_user_targets( + monkeypatch: pytest.MonkeyPatch, overrides, prisma_kwargs, expected_target +): + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma(**prisma_kwargs) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval(_start_request(**overrides), ADMIN) + + target_type, target_id, target_alias = expected_target + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert [(row["target_type"], row["target_id"]) for row in rows] == [(target_type, target_id)] + assert response.status == "running" + target = response.targets[0] + assert (target.target_type, target.target_id, target.target_alias, target.key_name) == ( + target_type, + target_id, + target_alias, + None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "overrides,prisma_kwargs,expected_detail", + [ + ( + {"api_key_ids": (), "team_ids": ("team-eng", "team-ghost")}, + {"known_teams": {"team-eng": "Engineering"}}, + "team_ids not on this proxy: team-ghost", + ), + ( + {"api_key_ids": (), "user_ids": ("dev-alice", "dev-ghost")}, + {"known_users": {"dev-alice": "alice@example.com"}}, + "user_ids not on this proxy: dev-ghost", + ), + ], + ids=["unknown-team", "unknown-user"], +) +async def test_start_shadow_eval_rejects_teams_and_users_this_proxy_does_not_know( + monkeypatch: pytest.MonkeyPatch, overrides, prisma_kwargs, expected_detail +): + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma(**prisma_kwargs) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(**overrides), ADMIN) + assert exc.value.status_code == 400 + assert expected_detail in exc.value.detail + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_start_shadow_eval_mixed_targets_create_both_legs_and_sweep_once_per_type( + monkeypatch: pytest.MonkeyPatch, +): + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma(known_teams={"team-eng": "Engineering"}) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval(_start_request(team_ids=("team-eng",)), ADMIN) + + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert [(row["target_type"], row["target_id"]) for row in rows] == [("key", "key-hash"), ("team", "team-eng")] + assert len({row["group_id"] for row in rows}) == 1 + sweeps = [ + call.args + for call in prisma.db.execute_raw.await_args_list + if "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in call.args[0] + ] + assert [(ids, target_type) for _, ids, target_type in sweeps] == [(["key-hash"], "key"), (["team-eng"], "team")] + assert [(t.target_type, t.target_id, t.target_alias) for t in response.targets] == [ + ("key", "key-hash", "prod-alpha"), + ("team", "team-eng", "Engineering"), + ] + + +@pytest.mark.asyncio +async def test_start_shadow_eval_names_the_busy_team_target(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma( + legs=[_leg_record(id="leg-t", group_id="job-7", target_type="team", target_id="team-eng")], + known_teams={"team-eng": "Engineering"}, + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(api_key_ids=(), team_ids=("team-eng",)), ADMIN) + assert exc.value.status_code == 409 + assert "team team-eng (job job-7)" in exc.value.detail + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_start_shadow_eval_claim_matches_exact_target_pairs_not_bare_ids(monkeypatch: pytest.MonkeyPatch): + """A key whose hash happens to spell a team's id must not hold the team's slot: the + claim matches (target_type, target_id) pairs, never ids across kinds.""" + import litellm.proxy.proxy_server as proxy_server + + _configure_anthropic_sdk_judge(monkeypatch) + prisma = _shadow_prisma( + legs=[_leg_record(id="leg-k", group_id="job-7", target_type="key", target_id="team-eng")], + known_teams={"team-eng": "Engineering"}, + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + response = await start_shadow_eval(_start_request(api_key_ids=(), team_ids=("team-eng",)), ADMIN) + + assert response.status == "running" + prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_list_shadow_eval_jobs_rejects_a_lone_filter_half(monkeypatch: pytest.MonkeyPatch): + """target_type and target_id only mean anything together: a bare id could name a key + or a team, and a bare type filters nothing.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record()]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + with pytest.raises(HTTPException) as id_only: + await list_shadow_eval_jobs(VIEWER, target_type=None, target_id="key-hash", limit=50) + assert id_only.value.status_code == 400 + + with pytest.raises(HTTPException) as type_only: + await list_shadow_eval_jobs(VIEWER, target_type="key", target_id=None, limit=50) + assert type_only.value.status_code == 400 + prisma.db.query_raw.assert_not_called() + + @pytest.mark.asyncio async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch): import litellm.proxy.proxy_server as proxy_server @@ -1530,7 +1755,7 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke }, ] prisma = _shadow_prisma( - legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=50)], + legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2", max_turns=50)], agg_rows=tier_rows, by_leg_rows=leg_rows, ) @@ -1549,8 +1774,10 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke assert response.results.by_tier[0].shadow_win_rate_pct == 50.0 assert response.results.overall_shadow_win_rate_pct == 40.0 assert response.results.overall_tie_rate_pct == 20.0 - assert [(s.group, s.turn_count) for s in response.results.by_key] == [("key-hash", 6), ("key-hash-2", 4)] - assert response.results.by_key[0].shadow_win_rate_pct == 66.7 + verdicts_by_target = {(t.target_type, t.target_id): t.verdicts for t in response.targets} + assert verdicts_by_target[("key", "key-hash")].turn_count == 6 + assert verdicts_by_target[("key", "key-hash")].shadow_win_rate_pct == 66.7 + assert verdicts_by_target[("key", "key-hash-2")].turn_count == 4 agg_sql = next(call.args[0] for call in prisma.db.query_raw.await_args_list if "real_spend" in call.args[0]) assert agg_sql.count("FILTER (WHERE real_cost IS NOT NULL AND NOT real_cache_hit)") == 2 assert response.results.by_tier[0].real_spend == 0.08 @@ -1561,7 +1788,10 @@ async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monke assert response.results.not_sampled_count is None assert response.results.unjudgeable_count is None assert response.results.shed_count is None - assert [(key.api_key_id, key.max_turns) for key in response.keys] == [("key-hash", 200), ("key-hash-2", 50)] + assert [(target.target_id, target.max_turns) for target in response.targets] == [ + ("key-hash", 200), + ("key-hash-2", 50), + ] totals_args = [call.args for call in prisma.db.query_raw.await_args_list if "judged_count" in call.args[0]] assert totals_args == [(totals_args[0][0], ["leg-1", "leg-2"])] error_where = prisma.db.litellm_shadowevalattempt.find_first.call_args.kwargs["where"] @@ -1595,7 +1825,7 @@ async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monke _leg_record(created_at=datetime(2026, 8, 13, tzinfo=timezone.utc)), _leg_record( id="leg-2", - api_key_id="key-hash-2", + target_id="key-hash-2", stopped_at=stamp, created_at=datetime(2026, 8, 13, tzinfo=timezone.utc), ), @@ -1615,14 +1845,14 @@ async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monke ) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) assert [(job.job_id, job.status) for job in jobs] == [ ("job-1", "running"), ("job-2", "stopped"), ("job-3", "completed"), ] - assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"] + assert [target.target_id for target in jobs[0].targets] == ["key-hash", "key-hash-2"] assert all(job.judged_count is None and job.results is None for job in jobs) legs_sql, legs_limit = prisma.db.query_raw.await_args_list[0].args assert "GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int" in legs_sql @@ -1648,17 +1878,20 @@ async def test_list_shadow_eval_jobs_filters_to_jobs_containing_the_key(monkeypa prisma = _shadow_prisma( legs=[ _leg_record(), - _leg_record(id="leg-2", api_key_id="key-hash-2"), - _leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash-2"), + _leg_record(id="leg-2", target_id="key-hash-2"), + _leg_record(id="leg-3", group_id="job-2", target_id="key-hash-2"), _leg_record(id="leg-4", group_id="job-3"), ] ) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id="key-hash-2", limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type="key", target_id="key-hash-2", limit=50) assert [job.job_id for job in jobs] == ["job-1", "job-2"] - assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"] + assert [target.target_id for target in jobs[0].targets] == ["key-hash", "key-hash-2"] + legs_sql, *legs_params = prisma.db.query_raw.await_args_list[0].args + assert "WHERE target_type = $2 AND target_id = $3" in legs_sql + assert legs_params == [50, "key", "key-hash-2"] @pytest.mark.parametrize( @@ -1682,7 +1915,7 @@ async def test_job_status_runs_until_every_key_stops_and_completed_outranks_stop legs=[ _leg_record( id=f"leg-{index}", - api_key_id=f"key-{index}", + target_id=f"key-{index}", stopped_at=stamp if stopped else None, ends_at=datetime.now(timezone.utc) + timedelta(days=days_left), ) @@ -1691,7 +1924,7 @@ async def test_job_status_runs_until_every_key_stops_and_completed_outranks_stop ) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) assert [job.status for job in jobs] == [expected] @@ -1707,9 +1940,9 @@ async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch prisma = _shadow_prisma( legs=[ _leg_record(max_turns=5), - _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=5), - _leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=5), - _leg_record(id="leg-4", group_id="job-2", api_key_id="key-hash-2", max_turns=5), + _leg_record(id="leg-2", target_id="key-hash-2", max_turns=5), + _leg_record(id="leg-3", group_id="job-2", target_id="key-hash", max_turns=5), + _leg_record(id="leg-4", group_id="job-2", target_id="key-hash-2", max_turns=5), ] ) prisma.attempt_rows = [ @@ -1720,13 +1953,13 @@ async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch ] monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) by_id = {job.job_id: job for job in jobs} assert by_id["job-1"].status == "completed" - assert all(key.stopped_at is None for key in by_id["job-1"].keys) + assert all(target.stopped_at is None for target in by_id["job-1"].targets) assert by_id["job-2"].status == "running" - assert {key.api_key_id: key.attempt_count for key in by_id["job-2"].keys} == {"key-hash": 5, "key-hash-2": 3} + assert {t.target_id: t.attempt_count for t in by_id["job-2"].targets} == {"key-hash": 5, "key-hash-2": 3} @pytest.mark.asyncio @@ -1740,7 +1973,7 @@ async def test_recorded_operator_stop_outranks_budget_arithmetic(monkeypatch: py prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6, "spend": 0.0}] monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) assert jobs[0].status == "stopped" assert jobs[0].stopped_by == "admin" @@ -1760,7 +1993,7 @@ async def test_backfilled_legacy_stop_never_reads_as_completion(monkeypatch: pyt prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6, "spend": 0.0}] monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) assert jobs[0].status == "stopped" @@ -1808,6 +2041,56 @@ def test_max_budget_migration_is_additive_and_leaves_legacy_rows_null(): @pytest.mark.asyncio +@pytest.mark.asyncio +async def test_verdicts_keep_same_id_targets_of_different_kinds_distinct(monkeypatch): + """A team and a user can legitimately share an id; their slices must not merge.""" + from litellm.proxy import proxy_server + + leg_rows = [ + { + "grp": "leg-1", + "turn_count": 6, + "real_wins": 2, + "shadow_wins": 4, + "ties": 0, + "avg_confidence": 0.8, + "real_spend": 0.02, + "shadow_spend": 0.01, + "cache_hit_turns": 0, + }, + { + "grp": "leg-2", + "turn_count": 4, + "real_wins": 3, + "shadow_wins": 0, + "ties": 1, + "avg_confidence": 0.6, + "real_spend": 0.05, + "shadow_spend": 0.04, + "cache_hit_turns": 1, + }, + ] + prisma = _shadow_prisma( + legs=[ + _leg_record(target_type="team", target_id="dev-alice"), + _leg_record(id="leg-2", target_type="user", target_id="dev-alice"), + ], + agg_rows=leg_rows[:1], + by_leg_rows=leg_rows, + known_teams={"dev-alice": "alias"}, + known_users={"dev-alice": "alice@example.com"}, + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + response = await get_shadow_eval_job("job-1", VIEWER) + + verdicts_by_target = {(t.target_type, t.target_id): t.verdicts for t in response.targets} + assert verdicts_by_target[("team", "dev-alice")].turn_count == 6 + assert verdicts_by_target[("team", "dev-alice")].shadow_win_rate_pct == 66.7 + assert verdicts_by_target[("user", "dev-alice")].turn_count == 4 + assert verdicts_by_target[("user", "dev-alice")].shadow_win_rate_pct == 0.0 + + async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pytest.MonkeyPatch): import litellm.proxy.proxy_server as proxy_server @@ -1832,9 +2115,9 @@ async def test_list_reads_completed_once_every_key_spends_its_dollar_budget(monk prisma = _shadow_prisma( legs=[ _leg_record(max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0), - _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0), + _leg_record(id="leg-2", target_id="key-hash-2", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0), _leg_record( - id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0 + id="leg-3", group_id="job-2", target_id="key-hash", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0 ), ] ) @@ -1845,13 +2128,13 @@ async def test_list_reads_completed_once_every_key_spends_its_dollar_budget(monk ] monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) by_id = {job.job_id: job for job in jobs} assert by_id["job-1"].status == "completed" assert by_id["job-2"].status == "running" - assert {key.api_key_id: key.spend for key in by_id["job-1"].keys} == {"key-hash": 1.0, "key-hash-2": 1.25} - assert all(key.max_budget == 1.0 for key in by_id["job-1"].keys) + assert {t.target_id: t.spend for t in by_id["job-1"].targets} == {"key-hash": 1.0, "key-hash-2": 1.25} + assert all(target.max_budget == 1.0 for target in by_id["job-1"].targets) @pytest.mark.asyncio @@ -1880,11 +2163,11 @@ async def test_legacy_jobs_without_a_dollar_budget_stay_turn_gated(monkeypatch: prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 40, "spend": 250.0}] monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) assert jobs[0].status == "running" - assert jobs[0].keys[0].max_budget is None - assert jobs[0].keys[0].spend == 250.0 + assert jobs[0].targets[0].max_budget is None + assert jobs[0].targets[0].spend == 250.0 @pytest.mark.asyncio @@ -1892,14 +2175,14 @@ async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest import litellm.proxy.proxy_server as proxy_server prisma = _shadow_prisma( - legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="deleted-key-hash")], + legs=[_leg_record(), _leg_record(id="leg-2", target_id="deleted-key-hash")], known_keys=("key-hash", "key-hash-2"), ) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) - assert [(key.key_alias, key.key_name) for key in jobs[0].keys] == [ + jobs = await list_shadow_eval_jobs(VIEWER, target_type=None, target_id=None, limit=50) + assert [(target.target_alias, target.key_name) for target in jobs[0].targets] == [ (None, None), ("prod-alpha", "sk-...lpha"), ] @@ -1907,7 +2190,7 @@ async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest assert batched_where == {"token": {"in": ["deleted-key-hash", "key-hash"]}} detail = await get_shadow_eval_job("job-1", VIEWER) - assert [key.key_alias for key in detail.keys] == [None, "prod-alpha"] + assert [target.target_alias for target in detail.targets] == [None, "prod-alpha"] @pytest.mark.asyncio @@ -1919,7 +2202,7 @@ async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_runnin import litellm.proxy.proxy_server as proxy_server earned = datetime.now(timezone.utc) - timedelta(hours=1) - prisma = _shadow_prisma(legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", stopped_at=earned)]) + prisma = _shadow_prisma(legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2", stopped_at=earned)]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) stopped = await stop_shadow_eval_job("job-1", ADMIN) @@ -1938,9 +2221,9 @@ async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_runnin assert datetime.fromisoformat(stop_stamp).tzinfo is None assert prisma.db.execute_raw.await_count == 1 prisma.db.litellm_shadowevaljob.update_many.assert_not_called() - by_key = {key.api_key_id: key.stopped_at for key in stopped.keys} - assert by_key["key-hash-2"] == earned - assert by_key["key-hash"] is not None and by_key["key-hash"] != earned + by_target = {target.target_id: target.stopped_at for target in stopped.targets} + assert by_target["key-hash-2"] == earned + assert by_target["key-hash"] is not None and by_target["key-hash"] != earned done_leg = _leg_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1)) prisma_done = _shadow_prisma(legs=[done_leg]) @@ -2331,7 +2614,7 @@ async def test_get_shadow_eval_job_sums_funnel_rows_across_legs(monkeypatch: pyt }, ] prisma = _shadow_prisma( - legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2")], + legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2")], agg_rows=tier_rows, ) prisma.funnel_rows = [{"legs_with_rows": 2, "not_sampled": 30, "unjudgeable": 5, "shed": 2, "withheld": 3}] @@ -2366,7 +2649,7 @@ async def test_partially_seeded_funnel_reads_as_unknown_coverage(monkeypatch: py }, ] prisma = _shadow_prisma( - legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2")], + legs=[_leg_record(), _leg_record(id="leg-2", target_id="key-hash-2")], agg_rows=tier_rows, ) prisma.funnel_rows = [{"legs_with_rows": 1, "not_sampled": 30, "unjudgeable": 5, "shed": 2, "withheld": 0}] diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index ef6e224761d..8b397f20552 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -39,6 +39,35 @@ vi.mock("@/app/(dashboard)/hooks/keys/useKeys", () => ({ })), })); +vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ + useInfiniteTeams: vi.fn(() => ({ + data: { pages: [{ teams: [{ team_id: "team-eng", team_alias: "engineering" }], page: 1, total_pages: 1 }] }, + isLoading: false, + fetchNextPage: vi.fn(), + hasNextPage: false, + isFetchingNextPage: false, + })), +})); + +vi.mock("@/app/(dashboard)/hooks/users/useUsers", () => ({ + useInfiniteUsers: vi.fn(() => ({ + data: { + pages: [ + { + users: [{ user_id: "dev-alice", user_alias: null, user_email: "alice@example.com" }], + page: 1, + total_pages: 1, + }, + ], + }, + isPending: false, + isError: false, + fetchNextPage: vi.fn(), + hasNextPage: false, + isFetchingNextPage: false, + })), +})); + vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ useAutoRouters: vi.fn(() => ({ data: [ @@ -60,7 +89,7 @@ vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ })), })); -import ShadowEvalSection, { shadowedKeyLabel } from "./ShadowEvalSection"; +import ShadowEvalSection, { shadowedTargetLabel } from "./ShadowEvalSection"; import { useShadowEvalJob, useShadowEvalJobs, @@ -77,14 +106,15 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ baseline_model: null, judge_model: "anthropic/claude-sonnet-5", shadow_percentage: 10, - keys: [ + targets: [ { - api_key_id: "hashed-key-abc", + target_type: "key", + target_id: "hashed-key-abc", max_turns: 10000, max_budget: 10, spend: 3.21, stopped_at: null, - key_alias: "prod-alpha", + target_alias: "prod-alpha", key_name: "sk-...alpha", }, ], @@ -129,7 +159,6 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ cache_hit_turns: 2, }, ], - by_key: [], overall_shadow_win_rate_pct: 48.0, overall_tie_rate_pct: 22.0, sampled_real_spend: 0.6, @@ -144,17 +173,18 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ ...overrides, }); -const keyEntry = ( - api_key_id: string, - overrides: Partial = {}, -): ShadowEvalJob["keys"][number] => ({ - api_key_id, +const targetEntry = ( + target_id: string, + overrides: Partial = {}, +): ShadowEvalJob["targets"][number] => ({ + target_type: "key", + target_id, max_turns: 10000, max_budget: 10, spend: 0, stopped_at: null, attempt_count: null, - key_alias: null, + target_alias: null, key_name: null, ...overrides, }); @@ -235,8 +265,8 @@ describe("ShadowEvalSection", () => { it("gives every active job its own card with a stop button, with the form still offered", () => { mockHooks({ jobs: [ - job({ job_id: "job-a", status: "running", keys: [keyEntry("key-a")] }), - job({ job_id: "job-b", status: "running", keys: [keyEntry("key-b")] }), + job({ job_id: "job-a", status: "running", targets: [targetEntry("key-a")] }), + job({ job_id: "job-b", status: "running", targets: [targetEntry("key-b")] }), ], }); render(); @@ -347,7 +377,7 @@ describe("ShadowEvalSection", () => { }); it("shows spend without a budget cap for a job from before spend budgets existed", () => { - const j = job({ keys: [keyEntry("hashed-key-abc", { max_budget: null, spend: 3.21 })] }); + const j = job({ targets: [targetEntry("hashed-key-abc", { max_budget: null, spend: 3.21 })] }); mockHooks({ jobs: [j], detailsById: { "job-1": j } }); render(); expect(screen.getByText(/\$3\.21 eval spend/)).toBeInTheDocument(); @@ -417,6 +447,38 @@ describe("ShadowEvalSection", () => { const expectedBody = { api_key_ids: ["hash-alpha", "hash-beta"], + team_ids: [], + user_ids: [], + router_name: "gpt-auto", + direction: "forward", + shadow_percentage: 10, + duration_days: 7, + max_budget: 10, + judge_model: "anthropic/claude-sonnet-5", + }; + expect(start.mutate).toHaveBeenCalledWith(expectedBody); + }); + + it("submits a team-only job with team_ids and no keys", async () => { + const user = userEvent.setup(); + const { start } = mockHooks({}); + render(); + + expect(screen.getByText("Start shadow eval")).toBeDisabled(); + + await user.click(screen.getByPlaceholderText("Search teams by alias")); + const teamList = await screen.findByTestId("paginated-multi-select-list"); + await user.click(within(teamList).getByText("engineering")); + await user.click(screen.getByPlaceholderText("Select an auto-router")); + await user.click(await screen.findByText("gpt-auto")); + await user.click(screen.getByPlaceholderText("Select a judge model")); + await user.click(await screen.findByRole("option", { name: /anthropic\/claude-sonnet-5/ })); + await user.click(screen.getByText("Start shadow eval")); + + const expectedBody = { + api_key_ids: [], + team_ids: ["team-eng"], + user_ids: [], router_name: "gpt-auto", direction: "forward", shadow_percentage: 10, @@ -453,6 +515,8 @@ describe("ShadowEvalSection", () => { const expectedBody = { api_key_ids: ["hash-alpha"], + team_ids: [], + user_ids: [], router_name: "gpt-auto", direction: "reverse", baseline_model: "prod-claude", @@ -485,9 +549,13 @@ describe("ShadowEvalSection", () => { }); it("labels the shadowed key by alias, then masked name, then truncated hash", () => { - expect(shadowedKeyLabel(job().keys[0])).toBe("prod-alpha"); - expect(shadowedKeyLabel(keyEntry("hashed-key-abc", { key_name: "sk-...alpha" }))).toBe("sk-...alpha"); - expect(shadowedKeyLabel(keyEntry("hashed-key-abc"))).toBe("hashed-key…"); + expect(shadowedTargetLabel(job().targets[0])).toBe("prod-alpha"); + expect(shadowedTargetLabel(targetEntry("hashed-key-abc", { key_name: "sk-...alpha" }))).toBe("sk-...alpha"); + expect(shadowedTargetLabel(targetEntry("hashed-key-abc"))).toBe("hashed-key…"); + expect(shadowedTargetLabel(targetEntry("team-eng", { target_type: "team" }))).toBe("team-eng"); + expect(shadowedTargetLabel(targetEntry("team-eng", { target_type: "team", target_alias: "engineering" }))).toBe( + "engineering", + ); }); it("breaks results down per key, so one key exhausting its own budget is visible while a sibling runs on", () => { @@ -495,15 +563,12 @@ describe("ShadowEvalSection", () => { jobs: [ job({ judged_count: 205, - keys: [ - keyEntry("hash-spent", { max_budget: 2, spend: 1.5, stopped_at: "2026-08-08T00:00:00Z" }), - keyEntry("hash-hungry", { max_budget: 5, spend: 0.2 }), - ], - results: { - by_tier: [], - by_current_model: [], - by_key: [ - { + targets: [ + targetEntry("hash-spent", { + max_budget: 2, + spend: 1.5, + stopped_at: "2026-08-08T00:00:00Z", + verdicts: { group: "hash-spent", turn_count: 200, real_win_rate_pct: 20.0, @@ -514,7 +579,12 @@ describe("ShadowEvalSection", () => { shadow_spend: 0.5, cache_hit_turns: 0, }, - ], + }), + targetEntry("hash-hungry", { max_budget: 5, spend: 0.2 }), + ], + results: { + by_tier: [], + by_current_model: [], overall_shadow_win_rate_pct: 60.0, overall_tie_rate_pct: 20.0, sampled_real_spend: 0.9, @@ -539,7 +609,7 @@ describe("ShadowEvalSection", () => { expect(screen.getByText(/205 turns judged/)).toBeInTheDocument(); expect(screen.getByText(/Shadowing 10% of/)).toBeInTheDocument(); - expect(screen.getByText("2 keys")).toBeInTheDocument(); + expect(screen.getByText("2 targets")).toBeInTheDocument(); }); it("reads a key that spent its budget as completed even before the sweep stamps it", () => { @@ -547,9 +617,9 @@ describe("ShadowEvalSection", () => { mockHooks({ jobs: [ job({ - keys: [ - keyEntry("hash-spent", { max_budget: 2, spend: 2, attempt_count: 40 }), - keyEntry("hash-hungry", legacyTurnBudgetLeg), + targets: [ + targetEntry("hash-spent", { max_budget: 2, spend: 2, attempt_count: 40 }), + targetEntry("hash-hungry", legacyTurnBudgetLeg), ], }), ], @@ -571,9 +641,9 @@ describe("ShadowEvalSection", () => { job({ judged_count: 0, results: null, - keys: [ - keyEntry("hash-spent", { max_budget: 0.5, spend: 0.5, attempt_count: 2 }), - keyEntry("hash-hungry", { max_budget: 5, spend: 0.01, attempt_count: 1 }), + targets: [ + targetEntry("hash-spent", { max_budget: 0.5, spend: 0.5, attempt_count: 2 }), + targetEntry("hash-hungry", { max_budget: 5, spend: 0.01, attempt_count: 1 }), ], }), ], @@ -594,9 +664,9 @@ describe("ShadowEvalSection", () => { jobs: [ job({ status: "completed", - keys: [ - keyEntry("hash-spent", { max_turns: 200, stopped_at: "2026-08-08T00:00:00Z" }), - keyEntry("hash-hungry", { max_turns: 500 }), + targets: [ + targetEntry("hash-spent", { max_turns: 200, stopped_at: "2026-08-08T00:00:00Z" }), + targetEntry("hash-hungry", { max_turns: 500 }), ], }), ], diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx index 39dee28390a..ec145bc617a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx @@ -3,10 +3,13 @@ import React, { useMemo, useState } from "react"; import { useInfiniteKeys } from "@/app/(dashboard)/hooks/keys/useKeys"; +import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import { useAutoRouters, usePlainModelGroups } from "@/app/(dashboard)/hooks/models/useModels"; import { PaginatedMultiSelect } from "@/components/shared/PaginatedMultiSelect"; +import TeamMultiSelect from "@/components/common_components/team_multi_select"; +import { userOptionLabel } from "@/components/common_components/UserDropdown"; import { SearchSelect, type SearchSelectOption } from "@/components/shared/SearchSelect"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -27,7 +30,7 @@ import { useStartShadowEval, useStopShadowEval, type ShadowEvalJob, - type ShadowEvalJobKey, + type ShadowEvalJobTarget, type ShadowEvalSlice, } from "./useShadowEval"; @@ -66,29 +69,31 @@ const routerMatchedOrBeatPct = ( ? 100 - results.overall_shadow_win_rate_pct : results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct; -export const shadowedKeyLabel = (key: ShadowEvalJobKey): string => - key.key_alias || key.key_name || `${key.api_key_id.slice(0, 10)}…`; +export const shadowedTargetLabel = (target: ShadowEvalJobTarget): string => + target.target_alias || + target.key_name || + (target.target_type === "key" ? `${target.target_id.slice(0, 10)}…` : target.target_id); -const shadowedKeysLabel = (job: ShadowEvalJob): string => - job.keys.length === 1 ? shadowedKeyLabel(job.keys[0]) : `${job.keys.length} keys`; +const shadowedTargetsLabel = (job: ShadowEvalJob): string => + job.targets.length === 1 ? shadowedTargetLabel(job.targets[0]) : `${job.targets.length} targets`; const totalBudget = (job: ShadowEvalJob): number | null => - job.keys.reduce( - (sum, key) => (sum === null || key.max_budget == null ? null : sum + key.max_budget), + job.targets.reduce( + (sum, target) => (sum === null || target.max_budget == null ? null : sum + target.max_budget), 0, ); -const totalSpend = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + (key.spend ?? 0), 0); +const totalSpend = (job: ShadowEvalJob): number => job.targets.reduce((sum, target) => sum + (target.spend ?? 0), 0); -const keySpent = (key: ShadowEvalJobKey): boolean => { - const spendBudgetReached = key.max_budget != null && key.spend != null && key.spend >= key.max_budget; - const turnValveReached = key.attempt_count != null && key.attempt_count >= key.max_turns; +const targetSpent = (target: ShadowEvalJobTarget): boolean => { + const spendBudgetReached = target.max_budget != null && target.spend != null && target.spend >= target.max_budget; + const turnValveReached = target.attempt_count != null && target.attempt_count >= target.max_turns; return spendBudgetReached || turnValveReached; }; -const keyStatus = (job: ShadowEvalJob, key: ShadowEvalJobKey): string => { - if (job.status === "completed" || (key.stopped_at == null && keySpent(key))) return "completed"; - return key.stopped_at != null ? "stopped" : "running"; +const targetStatus = (job: ShadowEvalJob, target: ShadowEvalJobTarget): string => { + if (job.status === "completed" || (target.stopped_at == null && targetSpent(target))) return "completed"; + return target.stopped_at != null ? "stopped" : "running"; }; const jobHeadline = (job: ShadowEvalJob): React.ReactNode => @@ -96,12 +101,12 @@ const jobHeadline = (job: ShadowEvalJob): React.ReactNode => <> Comparing {job.router_name} to{" "} {job.baseline_model} on {job.shadow_percentage}% of{" "} - {shadowedKeysLabel(job)} traffic + {shadowedTargetsLabel(job)} traffic ) : ( <> - Shadowing {job.shadow_percentage}% of {shadowedKeysLabel(job)} traffic - via {job.router_name} + Shadowing {job.shadow_percentage}% of {shadowedTargetsLabel(job)}{" "} + traffic via {job.router_name} ); @@ -255,13 +260,12 @@ const VerdictBar: React.FC<{ direction: ShadowEvalDirection; results: NonNullabl ); }; -const KeyTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => { - const slices = new Map((job.results?.by_key ?? []).map((slice) => [slice.group, slice])); +const TargetTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => { return ( - Key + Target Status {["Budget used", "Router wins", `${otherArmLabel(job.direction)} wins`].map((label) => ( @@ -271,18 +275,23 @@ const KeyTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => { - {job.keys.map((key) => { - const slice = slices.get(key.api_key_id); + {job.targets.map((target) => { + const slice = target.verdicts; return ( - - {shadowedKeyLabel(key)} + + + {shadowedTargetLabel(target)} + {target.target_type !== "key" && ( + {target.target_type} + )} + - + - {key.max_budget != null - ? `${usd(key.spend ?? 0)} / ${usd(key.max_budget)}` - : `${(key.attempt_count ?? slice?.turn_count ?? 0).toLocaleString()} / ${key.max_turns.toLocaleString()} turns`} + {target.max_budget != null + ? `${usd(target.spend ?? 0)} / ${usd(target.max_budget)}` + : `${(target.attempt_count ?? slice?.turn_count ?? 0).toLocaleString()} / ${target.max_turns.toLocaleString()} turns`} {slice ? ( <> @@ -318,9 +327,9 @@ const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({ const hasVerdicts = results != null && (results.by_tier.length > 0 || results.by_current_model.length > 0); return ( <> - {job.keys.length > 1 && ( + {job.targets.length > 1 && (
- +
)} {/* results == null re-stated for TS narrowing; hasVerdicts alone cannot narrow it */} @@ -454,9 +463,9 @@ const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[ const START_FORM_DESCRIPTION: Record = { forward: - "Duplicates a sampled slice of the selected keys' traffic through the auto-router and has an LLM judge compare both answers blind. Each key gets its own spend budget. The router's answers are never served to users; judge calls bill to the shadowed key.", + "Duplicates a sampled slice of the selected targets' traffic (keys, teams, or users) through the auto-router and has an LLM judge compare both answers blind. Each target gets its own spend budget. The router's answers are never served to users; judge calls bill to the sampled traffic's own identity.", reverse: - "Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each key gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the shadowed key.", + "Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each target gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the sampled traffic's own identity.", }; const DURATION_OPTIONS = [ @@ -515,9 +524,46 @@ const KeySelect: React.FC<{ value: string[]; onChange: (tokens: string[]) => voi ); }; +const UserSelect: React.FC<{ value: string[]; onChange: (ids: string[]) => void }> = ({ value, onChange }) => { + const [search, setSearch] = useState(""); + const { data, isPending, isError, fetchNextPage, hasNextPage, isFetchingNextPage } = useInfiniteUsers( + 50, + search || undefined, + ); + const options = useMemo( + () => + Array.from( + new Map( + (data?.pages ?? []) + .flatMap((page) => page.users) + .map((user) => [user.user_id, { label: userOptionLabel(user), value: user.user_id }] as const), + ).values(), + ), + [data], + ); + return ( + void fetchNextPage()} + hasNextPage={hasNextPage} + isFetchingNextPage={isFetchingNextPage} + isLoading={isPending} + placeholder="Search users by email" + emptyText="No matching users" + errorText={isError ? "Users could not be loaded. Refresh the page to retry." : undefined} + /> + ); +}; + const StartForm: React.FC = () => { const { accessToken } = useAuthorized(); const [apiKeyIds, setApiKeyIds] = useState([]); + const [teamIds, setTeamIds] = useState([]); + const [userIds, setUserIds] = useState([]); const [routerName, setRouterName] = useState(""); const [direction, setDirection] = useState("forward"); const [baselineModel, setBaselineModel] = useState(""); @@ -542,12 +588,15 @@ const StartForm: React.FC = () => { const parsedMaxBudget = Number.parseFloat(maxBudget); const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000; const baselinePicked = direction === "forward" || baselineModel !== ""; - const filled = apiKeyIds.length > 0 && [routerName, judgeModel].every((field) => field !== "") && baselinePicked; + const targetsPicked = apiKeyIds.length + teamIds.length + userIds.length > 0; + const filled = targetsPicked && [routerName, judgeModel].every((field) => field !== "") && baselinePicked; const boundsValid = percentageValid && maxBudgetValid; const valid = Boolean(accessToken) && filled && boundsValid; const handleStart = () => { const startBody = { api_key_ids: apiKeyIds, + team_ids: teamIds, + user_ids: userIds, router_name: routerName, direction, ...(direction === "reverse" ? { baseline_model: baselineModel } : {}), @@ -587,6 +636,12 @@ const StartForm: React.FC = () => { + + + + + + { value={maxBudget} onChange={(e) => setMaxBudget(e.target.value)} /> - max shadow + judge spend, per key + max shadow + judge spend, per target {maxBudget.trim() !== "" && !maxBudgetValid && (

Enter a value from 0.01 to 10000

@@ -774,8 +829,9 @@ const ShadowEvalSection: React.FC = () => {

Shadow eval

- Blind-judge the auto-router on your real traffic: against the models a key uses today before switching, or - against a fixed baseline after it has switched. + Blind-judge the auto-router on the real traffic of a key, team, or user (teams and users cover + JWT-authenticated traffic): against the models they use today before switching, or against a fixed baseline + after they have switched.

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts index eef98320e67..107df0f594a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts @@ -7,7 +7,7 @@ import { $api, fetchClient } from "@/lib/http/api"; import type { components } from "@/lib/http/schema"; export type ShadowEvalJob = components["schemas"]["ShadowEvalJobResponse"]; -export type ShadowEvalJobKey = components["schemas"]["ShadowEvalJobKeyResponse"]; +export type ShadowEvalJobTarget = components["schemas"]["ShadowEvalJobTargetResponse"]; export type ShadowEvalSlice = components["schemas"]["ShadowEvalSlice"]; export type StartShadowEvalRequest = components["schemas"]["StartShadowEvalRequest"]; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b79e2c12447..c20545f6fb2 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1234,8 +1234,8 @@ export interface paths { }; /** * List Shadow Eval Jobs - * @description List shadow eval jobs, newest first, each key with its attempt count so status is - * accurate. Judged counts, spend, and results ride the detail endpoint only. + * @description List shadow eval jobs, newest first, each target with its attempt count so status + * is accurate. Judged counts, spend, and results ride the detail endpoint only. */ get: operations["list_shadow_eval_jobs_auto_router_shadow_eval_get"]; put?: never; @@ -1257,22 +1257,29 @@ export interface paths { put?: never; /** * Start Shadow Eval - * @description Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against - * a second arm, judge the two responses blind, and stratify win rates by tier, by the model - * that served the real arm, and by key. + * @description Start a shadow eval: duplicate a sampled slice of one or more targets' live traffic + * against a second arm, judge the two responses blind, and stratify win rates by tier, + * by the model that served the real arm, and by target. * - * A forward job answers whether the keys should adopt router_name: it samples the requests - * the router did not serve and duplicates them through it. A reverse job answers whether a - * key already on the router still gains from it: it samples the requests the router did - * serve and duplicates them against baseline_model. A key can hold one active job per - * direction, so both questions can run at once. + * A target is a virtual key, a team, or a user. Team and user targets match on the + * identity every request resolves to at auth time, so they cover JWT-authenticated + * traffic, which presents no virtual key; a user target samples that user's traffic + * across all their teams, whether it arrives on a JWT or a key they own. * - * Shadow responses are never served to users. Each key samples until its recorded eval - * spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's - * window ends, or the job is stopped, so one key running out of budget does not end - * sampling for the others; sampling changes propagate to pods within about 10 seconds. - * Shadow and judge calls bill to the shadowed key but are excluded from request counts - * and auto-router adoption metrics. + * A forward job answers whether the targets should adopt router_name: it samples the + * requests the router did not serve and duplicates them through it. A reverse job + * answers whether a target already on the router still gains from it: it samples the + * requests the router did serve and duplicates them against baseline_model. A target + * can hold one active job per direction, so both questions can run at once, and a + * request matching several jobs' targets (say its key and its team) is sampled by + * each, separately budgeted. + * + * Shadow responses are never served to users. Each target samples until its recorded + * eval spend, the shadow and judge calls' own cost, reaches max_budget dollars, the + * job's window ends, or the job is stopped, so one target running out of budget does + * not end sampling for the others; sampling changes propagate to pods within about 10 + * seconds. Shadow and judge calls bill to the sampled request's own identity but are + * excluded from request counts and auto-router adoption metrics. */ post: operations["start_shadow_eval_auto_router_shadow_eval_start_post"]; delete?: never; @@ -1312,8 +1319,8 @@ export interface paths { put?: never; /** * Stop Shadow Eval Job - * @description Stop an active shadow eval job, every key it scopes at once. Attempts are kept; - * sampling halts within ~10s. Keys that already stopped on their own budget keep the + * @description Stop an active shadow eval job, every target it scopes at once. Attempts are kept; + * sampling halts within ~10s. Targets that already stopped on their own budget keep the * stopped_at they earned. The statement is the whole state machine: it claims the job * only while a leg still samples inside the window with no stop recorded, so a racing * operator, a same-instant budget spend, and a repeat stop all read the same 400 with @@ -35262,56 +35269,10 @@ export interface components { /** Timeout */ timeout?: number | null; }; - /** - * ShadowEvalJobKeyResponse - * @description One key a job shadows, with its own budget and stop state. - */ - ShadowEvalJobKeyResponse: { - /** - * Api Key Id - * @description The hashed virtual key whose traffic this entry scopes - */ - api_key_id: string; - /** - * Attempt Count - * @description This key's sampled attempts so far, judged and errored alike, the same count the sampler budgets against max_turns; populated on list and detail responses. Frozen at stopped_at once the key is stamped, so in-flight attempts landing after a stop never reclassify it - */ - attempt_count?: number | null; - /** - * Key Alias - * @description Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted - */ - key_alias?: string | null; - /** - * Key Name - * @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias - */ - key_name?: string | null; - /** - * Max Budget - * @description This key's own USD budget for the eval's shadow and judge spend, independent of its siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds - */ - max_budget?: number | null; - /** - * Max Turns - * @description This key's sample-count ceiling: the whole budget for jobs created before max_budget existed, and the error-loop safety valve otherwise - */ - max_turns: number; - /** - * Spend - * @description This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets against max_budget; populated on list and detail responses and frozen at stopped_at exactly like attempt_count - */ - spend?: number | null; - /** - * Stopped At - * @description When this key's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset - */ - stopped_at?: string | null; - }; /** * ShadowEvalJobResponse - * @description A shadow-eval job over one or more keys, each with its own budget and stop state; - * status is derived from stopped_by, the keys' stop and budget state, and ends_at, + * @description A shadow-eval job over one or more targets, each with its own budget and stop state; + * status is derived from stopped_by, the targets' stop and budget state, and ends_at, * never stored, so no writer anywhere can produce an inconsistent one. Aggregate * fields are populated by the detail endpoint only and stay None on list responses. */ @@ -35353,11 +35314,6 @@ export interface components { * @description Verdicts recorded; detail endpoint only */ judged_count?: number | null; - /** - * Keys - * @description The keys whose traffic this job evaluates, and only those keys', each with its own budget - */ - keys: components["schemas"]["ShadowEvalJobKeyResponse"][]; /** * Last Error * @description Most recent attempt error; detail endpoint only @@ -35373,8 +35329,8 @@ export interface components { * Status * @description Three recorded facts, no history-guessing: a stop is stopped_by (the migration * backfills it for every job that displayed stopped when the column arrived, so the - * pre-column population is closed), completion is the window passing or every key - * spending its budget, and anything else is running. The all-keys-stamped fallback + * pre-column population is closed), completion is the window passing or every target + * spending its budget, and anything else is running. The all-targets-stamped fallback * covers only stops written by pre-column pods during a rolling deploy. * @enum {string} */ @@ -35384,6 +35340,65 @@ export interface components { * @description The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled by migration for jobs that displayed stopped when the column arrived; None when the job ended on its own. Its presence is what makes a job read stopped rather than completed */ stopped_by?: string | null; + /** + * Targets + * @description The targets whose traffic this job evaluates, and only theirs, each with its own budget + */ + targets: components["schemas"]["ShadowEvalJobTargetResponse"][]; + }; + /** + * ShadowEvalJobTargetResponse + * @description One target a job shadows (a key, team, or user), with its own budget and stop state. + */ + ShadowEvalJobTargetResponse: { + /** + * Attempt Count + * @description This target's sampled attempts so far, judged and errored alike, the same count the sampler budgets against max_turns; populated on list and detail responses. Frozen at stopped_at once the target is stamped, so in-flight attempts landing after a stop never reclassify it + */ + attempt_count?: number | null; + /** + * Key Name + * @description Masked display name (sk-...) for key targets, resolved at read time; None for teams and users + */ + key_name?: string | null; + /** + * Max Budget + * @description This target's own USD budget for the eval's shadow and judge spend, independent of its siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds + */ + max_budget?: number | null; + /** + * Max Turns + * @description This target's sample-count ceiling: the whole budget for jobs created before max_budget existed, and the error-loop safety valve otherwise + */ + max_turns: number; + /** + * Spend + * @description This target's recorded shadow plus judge spend in USD, the same figure the sampler budgets against max_budget; populated on list and detail responses and frozen at stopped_at exactly like attempt_count + */ + spend?: number | null; + /** + * Stopped At + * @description When this target's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset + */ + stopped_at?: string | null; + /** + * Target Alias + * @description Display label resolved from the target's own row at read time: the key's alias, the team's alias, or the user's email; None when unset or deleted + */ + target_alias?: string | null; + /** + * Target Id + * @description The hashed virtual key, team id, or user id whose traffic this entry scopes + */ + target_id: string; + /** + * Target Type + * @description What kind of entity this entry scopes + * @enum {string} + */ + target_type: "key" | "team" | "user"; + /** @description This target's own judged-verdict slice; detail endpoint only, None until a turn is judged */ + verdicts?: components["schemas"]["ShadowEvalSlice"] | null; }; /** * ShadowEvalResult @@ -35395,11 +35410,6 @@ export interface components { * @description Sliced by the model that served the real arm: the keys' incumbent models in forward mode, and in reverse the models the router itself picked */ by_current_model: components["schemas"]["ShadowEvalSlice"][]; - /** - * By Key - * @description One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job scopes but has not judged a turn for yet are absent rather than reported as zero - */ - by_key: components["schemas"]["ShadowEvalSlice"][]; /** By Tier */ by_tier: components["schemas"]["ShadowEvalSlice"][]; /** @@ -35441,8 +35451,9 @@ export interface components { }; /** * ShadowEvalSlice - * @description Judge outcomes for one slice of a job's verdicts (a router tier, or one of the - * models that served the real arm). + * @description Judge outcomes for one slice of a job's verdicts: a router tier, one of the + * models that served the real arm, or one scoped target (embedded on that target's + * own entry, so slices never need re-joining to a target by id). */ ShadowEvalSlice: { /** Avg Judge Confidence */ @@ -35672,12 +35683,18 @@ export interface components { }; /** * StartShadowEvalRequest - * @description Start duplicating one or more keys' traffic for blind comparison against an auto-router. + * @description Start duplicating one or more targets' traffic for blind comparison against an auto-router. + * + * A target is a virtual key, a team, or a user; each becomes its own leg with its own + * budget and stop state. Team and user targets match on the identity every request + * carries after auth (user_api_key_team_id / user_api_key_user_id), so they cover + * JWT-authenticated traffic, which presents no virtual key at all. */ StartShadowEvalRequest: { /** * Api Key Ids - * @description The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these keys' traffic; requests made with any other key are not sampled. Each key carries its own max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 keys per job, which also bounds every read the job's endpoints make. + * @description Hashed virtual keys whose traffic will be shadowed. Combined with team_ids and user_ids the job needs at least one target and at most 100, which also bounds every read the job's endpoints make. Each target carries its own max_budget spend budget, so one exhausting its budget leaves the others sampling. + * @default [] */ api_key_ids: string[]; /** @@ -35706,7 +35723,7 @@ export interface components { judge_model: string; /** * Max Budget - * @description Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window + * @description Per-target USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped target samples until its recorded eval spend reaches this, so a job over N targets spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window * @default 10 */ max_budget: number; @@ -35717,9 +35734,21 @@ export interface components { router_name: string; /** * Shadow Percentage - * @description Percentage of the key's requests to duplicate through the router + * @description Percentage of each target's requests to duplicate through the router */ shadow_percentage: number; + /** + * Team Ids + * @description Teams whose traffic will be shadowed, matched on the team every authenticated request resolves to, so a team's JWT-auth and virtual-key traffic are both sampled + * @default [] + */ + team_ids: string[]; + /** + * User Ids + * @description Users whose traffic will be shadowed, matched on the user every authenticated request resolves to across all their teams: JWT requests carrying their subject claim and virtual keys they own + * @default [] + */ + user_ids: string[]; }; /** * SuccessfulKeyUpdate @@ -40681,8 +40710,10 @@ export interface operations { list_shadow_eval_jobs_auto_router_shadow_eval_get: { parameters: { query?: { - /** @description Filter to jobs that shadow this key, alone or alongside others */ - api_key_id?: string | null; + /** @description Kind of target to filter on; requires target_id */ + target_type?: ("key" | "team" | "user") | null; + /** @description Filter to jobs that shadow this target, alone or alongside others */ + target_id?: string | null; /** @description Newest jobs to return */ limit?: number; };