diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql new file mode 100644 index 00000000000..18ef5c40662 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql @@ -0,0 +1,7 @@ +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "group_id" TEXT; + +UPDATE "LiteLLM_ShadowEvalJob" SET "group_id" = "id" WHERE "group_id" IS NULL; + +ALTER TABLE "LiteLLM_ShadowEvalJob" ALTER COLUMN "group_id" SET NOT NULL; + +CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_group_id_idx" ON "LiteLLM_ShadowEvalJob"("group_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818000000_add_spend_log_timestamps/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818000000_add_spend_log_timestamps/migration.sql new file mode 100644 index 00000000000..a4a3cc3bb1b --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818000000_add_spend_log_timestamps/migration.sql @@ -0,0 +1,3 @@ +ALTER TABLE "LiteLLM_SpendLogs" +ADD COLUMN IF NOT EXISTS "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, +ADD COLUMN IF NOT EXISTS "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql new file mode 100644 index 00000000000..9efa3fdd052 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN "stopped_by" TEXT; + +UPDATE "LiteLLM_ShadowEvalJob" SET stopped_by = 'unknown' +WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc'); diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 24c0f1f11cc..f79e2bb0c18 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -641,6 +641,8 @@ model LiteLLM_SpendLogs { mcp_namespaced_tool_name String? agent_id String? proxy_server_request Json? @default("{}") + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") @@index([startTime]) @@index([startTime, request_id]) @@index([end_user]) @@ -1465,28 +1467,38 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } -// Shadow eval: evaluation of an auto-router against a key's live traffic, in either -// direction. forward duplicates the requests the key did not route through the router -// through it, answering whether the key should adopt it; reverse duplicates the requests -// the router did serve against a fixed baseline model, answering whether a key already on -// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge -// compares real vs shadow responses blind. The job row is immutable config plus -// stopped_at; every count, status, and spend figure is derived from the append-only -// attempt rows, so nothing can disagree across pods or stop races. +// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in +// either direction. forward duplicates the requests the keys did not route through the +// router through it, answering whether they should adopt it; reverse duplicates the +// requests the router did serve against a fixed baseline model, answering whether a key +// already on it still benefits. Either way a sampled slice runs in a detached task and an +// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job: +// immutable config plus that key's own turn budget and stop state, so one key exhausting +// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id +// (the id the API reports), written together by one atomic create_many with identical +// config; single-key jobs predating group_id were backfilled group_id = id. "One active +// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE +// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state +// partial indexes; it is what makes a concurrent start on another pod race-safe rather +// than read-then-create. Every count, status, and spend figure is derived from the +// append-only attempt rows, so nothing can disagree across pods or stop races. model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) - api_key_id String // hashed virtual key whose traffic is shadowed + group_id String // legs of one job share this; the API's job id + api_key_id String // hashed virtual key whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // sample budget: judge at most this many turns + max_turns Int // this key's sample budget: judge at most this many turns created_at DateTime @default(now()) created_by String? ends_at DateTime stopped_at DateTime? + stopped_by String? // operator who stopped it early; null when it ended on its own + @@index([group_id]) @@index([api_key_id]) @@index([created_at]) } diff --git a/litellm/models/spend_logs.py b/litellm/models/spend_logs.py index c5a0522864a..92b1a753ad5 100644 --- a/litellm/models/spend_logs.py +++ b/litellm/models/spend_logs.py @@ -33,6 +33,8 @@ class LiteLLM_SpendLogs(LiteLLMPydanticObjectBase): requester_ip_address: str | None = None messages: str | list | dict | None response: str | list | dict | None + created_at: datetime | None = None + updated_at: datetime | None = None class LiteLLM_ErrorLogs(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index d8b7414f32c..0112ad1f6ed 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -6,10 +6,13 @@ POST /auto_router/test_routing - Route one prompt through an unsaved complexity- from collections.abc import Mapping, Sequence from datetime import datetime, timedelta, timezone +from itertools import groupby +from operator import attrgetter from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Protocol +from uuid import uuid4 -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, field_validator from litellm._logging import verbose_proxy_logger from litellm.exceptions import BudgetExceededError @@ -41,6 +44,8 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( AutoRouterRoutingTestRequest, AutoRouterRoutingTestResponse, RequestComplexityRouterConfig, + ShadowEvalDirection, + ShadowEvalJobKeyResponse, ShadowEvalJobResponse, ShadowEvalResult, ShadowEvalSlice, @@ -89,17 +94,9 @@ class _ShadowEvalJobRow(Protocol): class _ShadowEvalJobTable(Protocol): - async def find_unique(self, *, where: Mapping[str, object]) -> _ShadowEvalJobRow | None: ... + async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ShadowEvalJobRow]: ... - async def find_first(self, *, where: Mapping[str, object]) -> _ShadowEvalJobRow | None: ... - - async def find_many( - self, *, where: Mapping[str, object], order: Mapping[str, str], take: int - ) -> Sequence[_ShadowEvalJobRow]: ... - - async def create(self, data: Mapping[str, object]) -> _ShadowEvalJobRow: ... - - async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> _ShadowEvalJobRow | None: ... + async def create_many(self, data: Sequence[Mapping[str, object]]) -> int: ... class _ShadowEvalAttemptRow(Protocol): @@ -606,18 +603,19 @@ _ATTEMPT_AGG_SELECT: Final = """ COUNT(*) FILTER (WHERE outcome = 'tie')::int AS ties, AVG(confidence)::float AS avg_confidence FROM "LiteLLM_ShadowEvalAttempt" -WHERE job_id = $1 AND outcome != 'error' +WHERE job_id = ANY($1::text[]) AND outcome != 'error' GROUP BY 1 """ _ATTEMPT_AGG_BY_TIER_SQL: Final = "SELECT COALESCE(tier, 'UNCLASSIFIED') AS grp," + _ATTEMPT_AGG_SELECT _ATTEMPT_AGG_BY_MODEL_SQL: Final = "SELECT COALESCE(real_model, 'unknown') AS grp," + _ATTEMPT_AGG_SELECT +_ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT _SWEEP_FINISHED_JOBS_SQL: Final = """ -UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = NOW() -WHERE j.api_key_id = $1 AND j.stopped_at IS NULL +UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = (NOW() AT TIME ZONE 'utc') +WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL AND ( - j.ends_at <= NOW() + j.ends_at <= (NOW() AT TIME ZONE 'utc') OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns ) """ @@ -628,7 +626,52 @@ SELECT COUNT(*) FILTER (WHERE outcome = 'error')::int AS error_count, COALESCE(SUM(judge_cost), 0)::float AS judge_spend FROM "LiteLLM_ShadowEvalAttempt" -WHERE job_id = $1 +WHERE job_id = ANY($1::text[]) +""" + +_ATTEMPT_COUNTS_SQL: Final = """ +SELECT a.job_id, COUNT(*)::int AS attempt_count +FROM "LiteLLM_ShadowEvalAttempt" a +JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id +WHERE a.job_id = ANY($1::text[]) AND (j.stopped_at IS NULL OR a.created_at <= j.stopped_at) +GROUP BY a.job_id +""" + +_STOP_JOB_SQL: Final = """ +UPDATE "LiteLLM_ShadowEvalJob" +SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp) +WHERE group_id = $1 AND stopped_by IS NULL + AND ends_at > (NOW() AT TIME ZONE 'utc') + AND EXISTS ( + SELECT 1 FROM "LiteLLM_ShadowEvalJob" k + WHERE k.group_id = $1 AND k.stopped_at IS NULL + AND (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_turns + ) +""" + + +class _AttemptCountRow(BaseModel): + job_id: str + attempt_count: int + + +_ATTEMPT_COUNT_ROWS: Final = TypeAdapter(list[_AttemptCountRow]) + + +_LIST_LEGS_SQL: Final = """ +SELECT * FROM "LiteLLM_ShadowEvalJob" +WHERE group_id IN ( + SELECT group_id FROM "LiteLLM_ShadowEvalJob" + GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int +) +""" + +_LIST_LEGS_BY_KEY_SQL: Final = """ +SELECT * FROM "LiteLLM_ShadowEvalJob" +WHERE group_id IN ( + SELECT group_id FROM "LiteLLM_ShadowEvalJob" WHERE api_key_id = $2 + GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int +) """ @@ -659,18 +702,98 @@ def _slices(rows: Sequence[_AttemptAggRow]) -> tuple[ShadowEvalSlice, ...]: ) +class _LegRow(BaseModel): + """One LiteLLM_ShadowEvalJob row, validated off the untyped prisma record. A row is + one key's leg of a job; the legs of a job share group_id and identical config, written + together by one create_many. The API's job id is the group id, so leg ids never leave + the server (attempts reference them internally).""" + + model_config = ConfigDict(from_attributes=True) + + id: str + group_id: str + api_key_id: str + router_name: str + direction: ShadowEvalDirection + baseline_model: str | None = None + judge_model: str + shadow_percentage: float + max_turns: int + created_at: datetime + ends_at: datetime + stopped_at: datetime | None = None + stopped_by: str | None = None + + @field_validator("created_at", "ends_at", "stopped_at") + @classmethod + def _as_aware_utc(cls, value: datetime | None) -> datetime | None: + """The columns store naive UTC wall time (prisma's convention); prisma reads hand + back aware datetimes while raw SQL reads hand back naive ones, so this boundary + makes every read aware UTC before anything compares or serializes them.""" + if value is None or value.tzinfo is not None: + return value + return value.replace(tzinfo=timezone.utc) + + +_LEG_ROWS: Final = TypeAdapter(list[_LegRow]) + + +async def _leg_attempt_counts(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> Mapping[str, int]: + """Each leg's attempt count by leg id, judged and errored alike, in one grouped read. + It is the same count the sampler budgets against max_turns, so the derived status + flips to completed exactly when sampling actually ends. A stamped leg's count freezes + at its stopped_at: in-flight attempts that land after the stamp are excluded, so they + can never reclassify a leg that was stopped under budget as budget-spent.""" + if not legs: + return MappingProxyType({}) + rows: Final = _ATTEMPT_COUNT_ROWS.validate_python( + await _query_raw(prisma_client, _ATTEMPT_COUNTS_SQL, [leg.id for leg in legs]) # mutable-ok: query param + or () + ) + return MappingProxyType({row.job_id: row.attempt_count for row in rows}) + + +def _group_response(group_id: str, legs: Sequence[_LegRow], attempt_counts: Mapping[str, int]) -> ShadowEvalJobResponse: + """The one constructor of a job response: the caller names the group and passes that + group's legs. Config is read off the first leg because every leg carries the same copy, + written by one create_many. No caller may serialize a raw row (that would leak a leg id + as the job id).""" + first: Final = legs[0] + return ShadowEvalJobResponse( + job_id=group_id, + keys=tuple( + ShadowEvalJobKeyResponse( + api_key_id=leg.api_key_id, + max_turns=leg.max_turns, + stopped_at=leg.stopped_at, + attempt_count=attempt_counts.get(leg.id, 0), + ) + for leg in sorted(legs, key=lambda leg: leg.api_key_id) + ), + router_name=first.router_name, + direction=first.direction, + baseline_model=first.baseline_model, + judge_model=first.judge_model, + shadow_percentage=first.shadow_percentage, + created_at=first.created_at, + ends_at=first.ends_at, + stopped_by=next((leg.stopped_by for leg in legs if leg.stopped_by is not None), None), + ) + + _NO_KEY_LABELS: Final[tuple[str | None, str | None]] = (None, None) async def _with_key_labels( prisma_client: "PrismaClient", responses: Sequence[ShadowEvalJobResponse] ) -> tuple[ShadowEvalJobResponse, ...]: - """Resolve each job's key hash to the key's alias and masked name in one batched read, + """Resolve every scoped key's hash to its alias and masked name in one batched read, so the UI can say whose traffic a job shadows. Deleted keys resolve to None.""" if not responses: return () + tokens: Final = sorted(frozenset(key.api_key_id for response in responses for key in response.keys)) key_rows: Final = await _verification_tokens(prisma_client).find_many( - where={"token": {"in": sorted({response.api_key_id for response in responses})}} # mutable-ok: Prisma filter + where={"token": {"in": tokens}} # mutable-ok: Prisma filter ) labels: Final[Mapping[str, tuple[str | None, str | None]]] = { row.token: (row.key_alias, row.key_name) for row in key_rows or () @@ -678,32 +801,50 @@ async def _with_key_labels( return tuple( response.model_copy( update={ # mutable-ok: pydantic update payload - "key_alias": labels.get(response.api_key_id, _NO_KEY_LABELS)[0], - "key_name": labels.get(response.api_key_id, _NO_KEY_LABELS)[1], + "keys": tuple( + key.model_copy( + update={ # mutable-ok: pydantic update payload + "key_alias": labels.get(key.api_key_id, _NO_KEY_LABELS)[0], + "key_name": labels.get(key.api_key_id, _NO_KEY_LABELS)[1], + } + ) + for key in response.keys + ) } ) for response in responses ) -async def _shadow_eval_results(prisma_client: "PrismaClient", job_id: str) -> ShadowEvalResult | None: - """Both stratifications of one job's verdicts. Tier answers "where does the router do - well"; the model stratification groups by whichever model served the real arm, so it - answers "which of the models this key uses today would the router beat" forward, and - "for the turns the router sent to X, did X beat the baseline" in reverse. Reads are - bounded by the job's own attempts (<= max_turns) via the job_id index.""" +async def _shadow_eval_results(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> ShadowEvalResult | None: + """All three stratifications of one job's verdicts. Tier answers "where does the router + do well"; the model stratification groups by whichever model served the real arm, so it + answers "which of the models these keys use today would the router beat" forward, and + "for the turns the router sent to X, did X beat the baseline" in reverse; key answers + "which key's traffic does the router suit". Reads are bounded by the job's own attempts + (<= the sum of its keys' max_turns) via the job_id index.""" + leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param by_tier: Final = _ATTEMPT_AGG_ROWS.validate_python( - await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, job_id) or () + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_TIER_SQL, leg_ids) or () ) if not by_tier: return None by_model: Final = _ATTEMPT_AGG_ROWS.validate_python( - await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, job_id) or () + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_MODEL_SQL, leg_ids) or () + ) + key_by_leg: Final = MappingProxyType({leg.id: leg.api_key_id for leg in legs}) + by_leg: Final = _ATTEMPT_AGG_ROWS.validate_python( + await _query_raw(prisma_client, _ATTEMPT_AGG_BY_LEG_SQL, leg_ids) or () + ) + by_key: Final = tuple( + row.model_copy(update={"grp": key_by_leg[row.grp]}) # mutable-ok: pydantic update payload + for row in by_leg ) total_turns: Final = sum(r.turn_count for r in by_tier) return ShadowEvalResult( by_tier=_slices(by_tier), by_current_model=_slices(by_model), + by_key=_slices(by_key), overall_shadow_win_rate_pct=_pct_of(sum(r.shadow_wins for r in by_tier), total_turns), overall_tie_rate_pct=_pct_of(sum(r.ties for r in by_tier), total_turns), ) @@ -721,20 +862,21 @@ async def start_shadow_eval( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> ShadowEvalJobResponse: """ - Start a shadow eval: duplicate a sampled slice of a key's live traffic against a second - arm, judge the two responses blind, and stratify win rates by tier and by the model that - served the real arm. + Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against + a second arm, judge the two responses blind, and stratify win rates by tier, by the model + that served the real arm, and by key. - A forward job answers whether the key should adopt router_name: it samples the requests + A forward job answers whether the keys should adopt router_name: it samples the requests the router did not serve and duplicates them through it. A reverse job answers whether a key already on the router still gains from it: it samples the requests the router did serve and duplicates them against baseline_model. A key can hold one active job per direction, so both questions can run at once. - Shadow responses are never served to users. The job samples until it has judged - max_turns turns, reaches the end of its window, or is stopped; sampling changes - propagate to pods within about 10 seconds. Shadow and judge calls bill to the - shadowed key but are excluded from request counts and auto-router adoption metrics. + Shadow responses are never served to users. Each key samples until it has judged + max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one + key running out of budget does not end sampling for the others; sampling changes + propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed + key but are excluded from request counts and auto-router adoption metrics. """ from litellm.proxy.proxy_server import llm_router, prisma_client @@ -746,48 +888,58 @@ async def start_shadow_eval( _validate_plain_model(llm_router, data.judge_model, "judge_model") if data.baseline_model is not None: _validate_plain_model(llm_router, data.baseline_model, "baseline_model") - key_row: Final = await _verification_tokens(prisma_client).find_unique( - where={"token": data.api_key_id} # mutable-ok: Prisma filter + token_rows: Final = await _verification_tokens(prisma_client).find_many( + where={"token": {"in": list(data.api_key_ids)}} # mutable-ok: Prisma filter ) - if key_row is None: + unknown: Final = tuple(sorted(frozenset(data.api_key_ids) - frozenset(row.token for row in token_rows or ()))) + if unknown: raise HTTPException( status_code=400, detail=( - f"api_key_id '{data.api_key_id}' is not a key on this proxy; pass the key's token hash, " + f"api_key_ids not on this proxy: {', '.join(unknown)}; pass each key's token hash, " "the value the key list and key info endpoints report" ), ) - # A job that expired or exhausted its turn budget stopped sampling on its own, but - # still holds its slot in the per-key, per-direction partial unique index until - # stamped; free it so a new eval can start. Sweeping both directions is deliberate. - await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, data.api_key_id) - active: Final = await _shadow_eval_jobs(prisma_client).find_first( + # A job whose window passed or whose turn budget ran out stopped sampling on its own, + # but its legs still hold their slots in the per-key, per-direction partial unique index + # until stamped; free them so a new eval can start. Sweeping both directions is deliberate. + requested: Final = list(data.api_key_ids) # mutable-ok: query param + await prisma_client.db.execute_raw(_SWEEP_FINISHED_JOBS_SQL, requested) + claimed: Final = await _shadow_eval_jobs(prisma_client).find_many( where={ # mutable-ok: Prisma filter - "api_key_id": data.api_key_id, + "api_key_id": {"in": requested}, # mutable-ok: Prisma filter "direction": data.direction, "stopped_at": None, }, ) - if active is not None: + if claimed: raise HTTPException( status_code=409, - detail=f"Key already has an active {data.direction} shadow eval job ({active.id}). Stop it first.", + detail=( + f"Already in an active {data.direction} shadow eval job: " + + ", ".join(sorted(f"{row.api_key_id} (job {row.group_id})" for row in claimed)) + + ". Stop it first." + ), ) now: Final = datetime.now(timezone.utc) + group_id: Final = str(uuid4()) + ends_at: Final = now + timedelta(days=data.duration_days) + shared_config: Final = { # mutable-ok: Prisma payload + "group_id": group_id, + "router_name": data.router_name, + "direction": data.direction, + "baseline_model": data.baseline_model, + "judge_model": data.judge_model, + "shadow_percentage": data.shadow_percentage, + "max_turns": data.max_turns, + "created_by": user_api_key_dict.user_id, + "created_at": now, + "ends_at": ends_at, + } try: - job: Final = await _shadow_eval_jobs(prisma_client).create( - data={ # mutable-ok: Prisma payload - "api_key_id": data.api_key_id, - "router_name": data.router_name, - "direction": data.direction, - "baseline_model": data.baseline_model, - "judge_model": data.judge_model, - "shadow_percentage": data.shadow_percentage, - "max_turns": data.max_turns, - "created_by": user_api_key_dict.user_id, - "ends_at": now + timedelta(days=data.duration_days), - } + await _shadow_eval_jobs(prisma_client).create_many( + data=[{**shared_config, "api_key_id": key} for key in data.api_key_ids] # mutable-ok: Prisma payload ) except Exception as e: if not _is_unique_violation(e): @@ -795,11 +947,28 @@ async def start_shadow_eval( raise HTTPException( status_code=409, detail=( - f"Key already has an active {data.direction} shadow eval job (started concurrently). Stop it first." + f"A requested key was claimed by another {data.direction} shadow eval job concurrently. Stop it first." ), ) from e - return ShadowEvalJobResponse.model_validate(job, from_attributes=True).model_copy( - update={"key_alias": key_row.key_alias, "key_name": key_row.key_name} # mutable-ok: pydantic update payload + labels: Final = MappingProxyType({row.token: row for row in token_rows}) + return ShadowEvalJobResponse( + job_id=group_id, + keys=tuple( + ShadowEvalJobKeyResponse( + api_key_id=api_key_id, + max_turns=data.max_turns, + key_alias=labels[api_key_id].key_alias, + key_name=labels[api_key_id].key_name, + ) + for api_key_id in sorted(data.api_key_ids) + ), + router_name=data.router_name, + direction=data.direction, + baseline_model=data.baseline_model, + judge_model=data.judge_model, + shadow_percentage=data.shadow_percentage, + created_at=now, + ends_at=ends_at, ) @@ -811,23 +980,38 @@ async def start_shadow_eval( ) async def list_shadow_eval_jobs( user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], - api_key_id: Annotated[str | None, Query(description="Filter to jobs shadowing this key")] = None, + api_key_id: Annotated[ + str | None, Query(description="Filter to jobs that shadow this key, alone or alongside others") + ] = None, limit: Annotated[int, Query(ge=1, le=200, description="Newest jobs to return")] = 50, ) -> tuple[ShadowEvalJobResponse, ...]: - """List shadow eval jobs, newest first. Counts and results ride the detail endpoint only.""" + """List shadow eval jobs, newest first, each key with its attempt count so status is + accurate. Judged counts, spend, and results ride the detail endpoint only.""" from litellm.proxy.proxy_server import prisma_client _require_admin_viewer(user_api_key_dict, "view shadow evals") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - records: Final = await _shadow_eval_jobs(prisma_client).find_many( - where={"api_key_id": api_key_id} if api_key_id else {}, # mutable-ok: Prisma filter - order={"created_at": "desc"}, # mutable-ok: Prisma order - take=limit, + legs: Final = _LEG_ROWS.validate_python( + ( + await _query_raw(prisma_client, _LIST_LEGS_BY_KEY_SQL, limit, api_key_id) + if api_key_id + else await _query_raw(prisma_client, _LIST_LEGS_SQL, limit) + ) + or () ) + by_group: Final[Mapping[str, tuple[_LegRow, ...]]] = MappingProxyType( + { + group_id: tuple(group) + for group_id, group in groupby(sorted(legs, key=attrgetter("group_id")), key=attrgetter("group_id")) + } + ) + newest_first: Final = sorted( + by_group, key=lambda group_id: max(leg.created_at for leg in by_group[group_id]), reverse=True + ) + counts: Final = await _leg_attempt_counts(prisma_client, legs) return await _with_key_labels( - prisma_client, - tuple(ShadowEvalJobResponse.model_validate(record, from_attributes=True) for record in records or ()), + prisma_client, tuple(_group_response(group_id, by_group[group_id], counts) for group_id in newest_first) ) @@ -847,20 +1031,24 @@ async def get_shadow_eval_job( _require_admin_viewer(user_api_key_dict, "view shadow evals") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - record: Final = await _shadow_eval_jobs(prisma_client).find_unique( - where={"id": job_id} # mutable-ok: Prisma filter + legs: Final = _LEG_ROWS.validate_python( + await _shadow_eval_jobs(prisma_client).find_many( + where={"group_id": job_id} # mutable-ok: Prisma filter + ) + or () ) - if record is None: + if not legs: raise HTTPException(status_code=404, detail=f"No shadow eval job {job_id}") + leg_ids: Final = [leg.id for leg in legs] # mutable-ok: query param totals: Final = _ATTEMPT_TOTALS_ROWS.validate_python( - await _query_raw(prisma_client, _ATTEMPT_TOTALS_SQL, job_id) or () + await _query_raw(prisma_client, _ATTEMPT_TOTALS_SQL, leg_ids) or () ) latest_error: Final = await _shadow_eval_attempts(prisma_client).find_first( - where={"job_id": job_id, "outcome": "error"}, # mutable-ok: Prisma filter + where={"job_id": {"in": leg_ids}, "outcome": "error"}, # mutable-ok: Prisma filter order={"created_at": "desc"}, # mutable-ok: Prisma order ) labeled: Final = await _with_key_labels( - prisma_client, (ShadowEvalJobResponse.model_validate(record, from_attributes=True),) + prisma_client, (_group_response(job_id, legs, await _leg_attempt_counts(prisma_client, legs)),) ) return labeled[0].model_copy( update={ # mutable-ok: pydantic update payload @@ -868,7 +1056,7 @@ async def get_shadow_eval_job( "error_count": totals[0].error_count if totals else 0, "judge_spend": round(totals[0].judge_spend, 6) if totals else 0.0, "last_error": latest_error.error if latest_error else None, - "results": await _shadow_eval_results(prisma_client, job_id), + "results": await _shadow_eval_results(prisma_client, legs), } ) @@ -883,25 +1071,33 @@ async def stop_shadow_eval_job( job_id: str, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ) -> ShadowEvalJobResponse: - """Stop an active shadow eval job. Attempts are kept; sampling halts within ~10s.""" + """Stop an active shadow eval job, every key it scopes at once. Attempts are kept; + sampling halts within ~10s. Keys that already stopped on their own budget keep the + stopped_at they earned. The statement is the whole state machine: it claims the job + only while a leg still samples inside the window with no stop recorded, so a racing + operator, a same-instant budget spend, and a repeat stop all read the same 400 with + the status the job actually holds.""" from litellm.proxy.proxy_server import prisma_client _require_admin_writer(user_api_key_dict, "stop a shadow eval") if prisma_client is None: raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) - record: Final = await _shadow_eval_jobs(prisma_client).find_unique( - where={"id": job_id} # mutable-ok: Prisma filter + stamp: Final = datetime.now(timezone.utc) + operator: Final = user_api_key_dict.user_id or "operator" + claimed: Final = await prisma_client.db.execute_raw( + _STOP_JOB_SQL, job_id, operator, stamp.replace(tzinfo=None).isoformat() ) - if record is None: + legs: Final = _LEG_ROWS.validate_python( + await _shadow_eval_jobs(prisma_client).find_many( + where={"group_id": job_id} # mutable-ok: Prisma filter + ) + or () + ) + if not legs: raise HTTPException(status_code=404, detail=f"No shadow eval job {job_id}") - current: Final = ShadowEvalJobResponse.model_validate(record, from_attributes=True) - if current.status != "running": + counts: Final = await _leg_attempt_counts(prisma_client, legs) + current: Final = _group_response(job_id, legs, counts) + if claimed == 0: raise HTTPException(status_code=400, detail=f"Job {job_id} is already {current.status}") - updated: Final = await _shadow_eval_jobs(prisma_client).update( - where={"id": job_id}, # mutable-ok: Prisma filter - data={"stopped_at": datetime.now(timezone.utc)}, # mutable-ok: Prisma payload - ) - labeled: Final = await _with_key_labels( - prisma_client, (ShadowEvalJobResponse.model_validate(updated, from_attributes=True),) - ) + labeled: Final = await _with_key_labels(prisma_client, (current,)) return labeled[0] diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 24c0f1f11cc..f79e2bb0c18 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -641,6 +641,8 @@ model LiteLLM_SpendLogs { mcp_namespaced_tool_name String? agent_id String? proxy_server_request Json? @default("{}") + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") @@index([startTime]) @@index([startTime, request_id]) @@index([end_user]) @@ -1465,28 +1467,38 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } -// Shadow eval: evaluation of an auto-router against a key's live traffic, in either -// direction. forward duplicates the requests the key did not route through the router -// through it, answering whether the key should adopt it; reverse duplicates the requests -// the router did serve against a fixed baseline model, answering whether a key already on -// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge -// compares real vs shadow responses blind. The job row is immutable config plus -// stopped_at; every count, status, and spend figure is derived from the append-only -// attempt rows, so nothing can disagree across pods or stop races. +// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in +// either direction. forward duplicates the requests the keys did not route through the +// router through it, answering whether they should adopt it; reverse duplicates the +// requests the router did serve against a fixed baseline model, answering whether a key +// already on it still benefits. Either way a sampled slice runs in a detached task and an +// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job: +// immutable config plus that key's own turn budget and stop state, so one key exhausting +// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id +// (the id the API reports), written together by one atomic create_many with identical +// config; single-key jobs predating group_id were backfilled group_id = id. "One active +// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE +// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state +// partial indexes; it is what makes a concurrent start on another pod race-safe rather +// than read-then-create. Every count, status, and spend figure is derived from the +// append-only attempt rows, so nothing can disagree across pods or stop races. model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) - api_key_id String // hashed virtual key whose traffic is shadowed + group_id String // legs of one job share this; the API's job id + api_key_id String // hashed virtual key whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // sample budget: judge at most this many turns + max_turns Int // this key's sample budget: judge at most this many turns created_at DateTime @default(now()) created_by String? ends_at DateTime stopped_at DateTime? + stopped_by String? // operator who stopped it early; null when it ended on its own + @@index([group_id]) @@index([api_key_id]) @@index([created_at]) } diff --git a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py index efdbda47fdc..25de9f6d065 100644 --- a/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py +++ b/litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py @@ -17,6 +17,7 @@ import json from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import date, datetime, time, timedelta, timezone +from types import MappingProxyType from typing import TYPE_CHECKING, Final from litellm._logging import verbose_proxy_logger @@ -110,17 +111,56 @@ def _public_model_name(row: object, model_info: Mapping[str, object]) -> str: def _decode_model_info(raw: object) -> "Mapping[str, object] | None": - """A deployment's model_info as a dict, decoding a JSON string, else None.""" + """A deployment's model_info as a mapping, decoding a JSON string, else None. + + Valid JSON that is not an object decodes to a list or a scalar, which every caller + would then read fields off, so it is rejected here rather than raised past them. + """ if isinstance(raw, str): try: - return json.loads(raw) + decoded: Final = json.loads(raw) except (TypeError, ValueError): return None - if isinstance(raw, dict): + return decoded if isinstance(decoded, dict) else None + if isinstance(raw, Mapping): return raw return None +@dataclass(frozen=True, slots=True) +class _PTUDeployment: + """A deployment in the shape ``_parse_ptu_model`` reads, whatever declared it. + + A ``LiteLLM_ProxyModelTable`` row already has it. A router entry does not: its id + lives in ``model_info.id`` rather than on the entry itself. + """ + + model_id: str + model_name: str + model_info: Mapping[str, object] + + +def _router_deployment(deployment: Mapping[str, object]) -> _PTUDeployment | None: + """A router ``model_list`` entry in the shape the parser reads, else None. + + An id is required rather than defaulted because it keys the sentinel row: every + deployment without one would collapse onto a single row per team and only the last + would be billed. The mapping is copied because the router rewrites entries in place + while the rollup runs. + """ + model_info: Final = _decode_model_info(deployment.get("model_info")) + if model_info is None: + return None + model_id: Final = model_info.get("id") + if not isinstance(model_id, str) or not model_id: + return None + return _PTUDeployment( + model_id=model_id, + model_name=str(deployment.get("model_name") or ""), + model_info=MappingProxyType(dict(model_info)), + ) + + def _parse_ptu_model(row: object) -> PTUModel | None: """Return a PTUModel when the deployment carries valid manual PTU config, else None. diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 9461297feca..63c93e0f268 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -6,7 +6,7 @@ from collections.abc import Mapping from datetime import datetime, timezone from typing import Final, Literal, TypeAlias -from pydantic import AliasChoices, BaseModel, ConfigDict, Field, computed_field, field_validator, model_validator +from pydantic import BaseModel, Field, computed_field, field_validator, model_validator from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig from litellm.types.utils import StandardLoggingRoutingDecision @@ -155,13 +155,17 @@ DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5" class StartShadowEvalRequest(BaseModel): - """Start duplicating a key's traffic for blind comparison against an auto-router.""" + """Start duplicating one or more keys' traffic for blind comparison against an auto-router.""" - api_key_id: str = Field( + api_key_ids: tuple[str, ...] = Field( + min_length=1, + max_length=100, description=( - "The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this " - "key's traffic; requests made with any other key are not sampled." - ) + "The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these " + "keys' traffic; requests made with any other key are not sampled. Each key carries its own " + "max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 " + "keys per job, which also bounds every read the job's endpoints make." + ), ) router_name: str = Field(description="The auto-router under evaluation, in either direction") direction: ShadowEvalDirection = Field( @@ -204,8 +208,9 @@ class StartShadowEvalRequest(BaseModel): ge=1, le=2000, description=( - "Sample budget: the job judges at most this many turns, then completes. This is also the spend " - "bound; expected judge cost is roughly max_turns times one judge call" + "Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, " + "so a job over N keys judges at most N times max_turns turns. This is also the spend bound; " + "expected judge cost is roughly that turn ceiling times one judge call" ), ) @@ -214,6 +219,12 @@ class StartShadowEvalRequest(BaseModel): def _round_percentage(cls, value: float) -> float: return round(value, 2) + @field_validator("api_key_ids") + @classmethod + def _dedupe_keys(cls, value: tuple[str, ...]) -> tuple[str, ...]: + """A key named twice would collide with itself on the one-active-per-(key, direction) index.""" + return tuple(dict.fromkeys(value)) + @model_validator(mode="after") def _baseline_model_matches_direction(self) -> "StartShadowEvalRequest": if self.direction == "reverse" and self.baseline_model is None: @@ -251,24 +262,46 @@ class ShadowEvalResult(BaseModel): by_tier: tuple[ShadowEvalSlice, ...] by_current_model: tuple[ShadowEvalSlice, ...] = Field( description=( - "Sliced by the model that served the real arm: the key's incumbent models in forward mode, " + "Sliced by the model that served the real arm: the keys' incumbent models in forward mode, " "and in reverse the models the router itself picked" ) ) + by_key: tuple[ShadowEvalSlice, ...] = Field( + description=( + "One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job " + "scopes but has not judged a turn for yet are absent rather than reported as zero" + ), + ) overall_shadow_win_rate_pct: float overall_tie_rate_pct: float -class ShadowEvalJobResponse(BaseModel): - """A shadow-eval job. Validates directly from the prisma record (job_id reads the - row's id); status is derived from stopped_at and ends_at, never stored, so no writer - anywhere can produce an inconsistent one. Aggregate fields are populated by the - detail endpoint only and stay None on list responses.""" +class ShadowEvalJobKeyResponse(BaseModel): + """One key a job shadows, with its own budget and stop state.""" - model_config = ConfigDict(from_attributes=True, populate_by_name=True) + api_key_id: str = Field(description="The hashed virtual key whose traffic this entry scopes") + max_turns: int = Field(description="This key's own sample budget, independent of its siblings'") + stopped_at: datetime | None = Field( + default=None, + description=( + "When this key's slot was stamped free, whether its own budget ran out, the window closed, " + "or an operator stopped the job; status is derived, so a spent budget reads completed even " + "while this is still unset" + ), + ) + attempt_count: int | None = Field( + default=None, + description=( + "This key's sampled attempts so far, judged and errored alike, the same count the sampler " + "budgets against max_turns; populated on list and detail responses. Frozen at stopped_at " + "once the key is stamped, so in-flight attempts landing after a stop never reclassify it" + ), + ) + + @property + def budget_spent(self) -> bool: + return self.attempt_count is not None and self.attempt_count >= self.max_turns - job_id: str = Field(validation_alias=AliasChoices("id", "job_id")) - api_key_id: str = Field(description="The hashed virtual key whose traffic this job evaluates, and only that key's") key_alias: str | None = Field( default=None, description="Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted", @@ -277,15 +310,34 @@ class ShadowEvalJobResponse(BaseModel): default=None, description="Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias", ) + + +class ShadowEvalJobResponse(BaseModel): + """A shadow-eval job over one or more keys, each with its own budget and stop state; + status is derived from stopped_by, the keys' stop and budget state, and ends_at, + never stored, so no writer anywhere can produce an inconsistent one. Aggregate + fields are populated by the detail endpoint only and stay None on list responses.""" + + job_id: str + keys: tuple[ShadowEvalJobKeyResponse, ...] = Field( + min_length=1, + description="The keys whose traffic this job evaluates, and only those keys', each with its own budget", + ) router_name: str direction: ShadowEvalDirection = "forward" baseline_model: str | None = None judge_model: str shadow_percentage: float - max_turns: int created_at: datetime ends_at: datetime - stopped_at: datetime | None = None + stopped_by: str | None = Field( + default=None, + description=( + "The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled " + "by migration for jobs that displayed stopped when the column arrived; None when the job " + "ended on its own. Its presence is what makes a job read stopped rather than completed" + ), + ) judged_count: int | None = Field(default=None, description="Verdicts recorded; detail endpoint only") error_count: int | None = Field(default=None, description="Sampled attempts that errored; detail endpoint only") @@ -296,12 +348,19 @@ class ShadowEvalJobResponse(BaseModel): @computed_field @property def status(self) -> ShadowEvalStatus: - """A job whose window has passed reads completed even if a later sweep stamped - stopped_at; stopped means sampling ended before the window did.""" + """Three recorded facts, no history-guessing: a stop is stopped_by (the migration + backfills it for every job that displayed stopped when the column arrived, so the + pre-column population is closed), completion is the window passing or every key + spending its budget, and anything else is running. The all-keys-stamped fallback + covers only stops written by pre-column pods during a rolling deploy.""" + if self.stopped_by is not None: + return "stopped" if datetime.now(timezone.utc) >= ( self.ends_at if self.ends_at.tzinfo else self.ends_at.replace(tzinfo=timezone.utc) ): return "completed" - if self.stopped_at is not None: + if all(key.budget_spent for key in self.keys): + return "completed" + if all(key.stopped_at is not None for key in self.keys): return "stopped" return "running" diff --git a/schema.prisma b/schema.prisma index 24c0f1f11cc..f79e2bb0c18 100644 --- a/schema.prisma +++ b/schema.prisma @@ -641,6 +641,8 @@ model LiteLLM_SpendLogs { mcp_namespaced_tool_name String? agent_id String? proxy_server_request Json? @default("{}") + created_at DateTime @default(now()) @map("created_at") + updated_at DateTime @default(now()) @updatedAt @map("updated_at") @@index([startTime]) @@index([startTime, request_id]) @@index([end_user]) @@ -1465,28 +1467,38 @@ model LiteLLM_AutoRouterSession { @@index([last_turn_at], map: "idx_autorouter_session_last_turn") } -// Shadow eval: evaluation of an auto-router against a key's live traffic, in either -// direction. forward duplicates the requests the key did not route through the router -// through it, answering whether the key should adopt it; reverse duplicates the requests -// the router did serve against a fixed baseline model, answering whether a key already on -// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge -// compares real vs shadow responses blind. The job row is immutable config plus -// stopped_at; every count, status, and spend figure is derived from the append-only -// attempt rows, so nothing can disagree across pods or stop races. +// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in +// either direction. forward duplicates the requests the keys did not route through the +// router through it, answering whether they should adopt it; reverse duplicates the +// requests the router did serve against a fixed baseline model, answering whether a key +// already on it still benefits. Either way a sampled slice runs in a detached task and an +// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job: +// immutable config plus that key's own turn budget and stop state, so one key exhausting +// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id +// (the id the API reports), written together by one atomic create_many with identical +// config; single-key jobs predating group_id were backfilled group_id = id. "One active +// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE +// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state +// partial indexes; it is what makes a concurrent start on another pod race-safe rather +// than read-then-create. Every count, status, and spend figure is derived from the +// append-only attempt rows, so nothing can disagree across pods or stop races. model LiteLLM_ShadowEvalJob { id String @id @default(cuid()) - api_key_id String // hashed virtual key whose traffic is shadowed + group_id String // legs of one job share this; the API's job id + api_key_id String // hashed virtual key whose traffic this leg shadows router_name String // the auto-router under evaluation, in either direction direction String @default("forward") // forward | reverse baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // sample budget: judge at most this many turns + max_turns Int // this key's sample budget: judge at most this many turns created_at DateTime @default(now()) created_by String? ends_at DateTime stopped_at DateTime? + stopped_by String? // operator who stopped it early; null when it ended on its own + @@index([group_id]) @@index([api_key_id]) @@index([created_at]) } diff --git a/tests/test_litellm/models/test_models.py b/tests/test_litellm/models/test_models.py index 786f6244930..187c7aa7f5e 100644 --- a/tests/test_litellm/models/test_models.py +++ b/tests/test_litellm/models/test_models.py @@ -498,6 +498,25 @@ class TestSpendLogs: assert log.request_id == "r1" assert log.spend == 0.0 assert log.cache_hit == "False" + assert log.created_at is None + assert log.updated_at is None + + def test_spend_logs_parse_database_timestamps(self): + created_at = datetime(2026, 8, 18, 12, 0, 0) + updated_at = datetime(2026, 8, 18, 12, 5, 0) + log = LiteLLM_SpendLogs( + request_id="r1", + api_key="sk-1", + call_type="completion", + startTime=None, + endTime=None, + messages=None, + response=None, + created_at=created_at, + updated_at=updated_at, + ) + assert log.created_at == created_at + assert log.updated_at == updated_at def test_error_logs_creation(self): log = LiteLLM_ErrorLogs( diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 77149457e82..3fd023552f5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -4,6 +4,7 @@ Unit tests for auto router management endpoints import os import sys +from pathlib import Path import pytest from fastapi import HTTPException @@ -325,9 +326,7 @@ class TestAutoRouterBenchmarks: from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals totals = _benchmark_totals(self.ROW) - bucket_hits = ( - totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits - ) + bucket_hits = totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits assert bucket_hits == 27 assert totals.cache.hit_rate_pct == pytest.approx(100.0 * 28 / 38, abs=0.1) @@ -490,7 +489,7 @@ from litellm.proxy.management_endpoints.auto_router_endpoints import ( start_shadow_eval, stop_shadow_eval_job, ) -from litellm.types.management_endpoints.auto_router_endpoints import ShadowEvalJobResponse, StartShadowEvalRequest +from litellm.types.management_endpoints.auto_router_endpoints import StartShadowEvalRequest VIEWER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, api_key="sk-view", user_id="viewer") NON_ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-user", user_id="user") @@ -507,19 +506,23 @@ def _shadow_router() -> MagicMock: return router -def _job_record(**overrides: object) -> MagicMock: - """Spec'd like a real prisma row: only the table's columns exist as attributes, so - from_attributes validation falls back to model defaults for everything else.""" +def _leg_record(**overrides: object) -> MagicMock: + """Spec'd like a real prisma row: only the table's columns exist as attributes. One + row is one key's leg of a job; legs sharing group_id are one job.""" defaults = { - "id": "job-1", + "id": "leg-1", + "group_id": "job-1", "api_key_id": "key-hash", "router_name": "my-router", + "direction": "forward", + "baseline_model": None, "judge_model": "anthropic/claude-sonnet-5", "shadow_percentage": 10.0, "max_turns": 200, "created_at": datetime(2026, 8, 11, tzinfo=timezone.utc), "ends_at": datetime.now(timezone.utc) + timedelta(days=7), "stopped_at": None, + "stopped_by": None, } fields = {**defaults, **overrides} record = MagicMock(spec=list(fields)) @@ -538,23 +541,90 @@ def _key_record( return record -def _shadow_prisma(active_job=None, agg_rows=None) -> MagicMock: +def _shadow_prisma(legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-hash", "key-hash-2")) -> MagicMock: + """The job-table fake honours the filters it is handed, so a read that forgets + stopped_at sees rows the partial index would have released, one that forgets + direction sees the opposite-direction legs a key may hold at the same time, and a + group read that matched on a leg id would come back empty.""" prisma = MagicMock() - prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=_key_record()) - prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record()]) - prisma.db.execute_raw = AsyncMock(return_value=0) - prisma.db.litellm_shadowevaljob.find_first = AsyncMock(return_value=active_job) - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=None) - prisma.db.litellm_shadowevaljob.find_many = AsyncMock(return_value=[]) - prisma.db.litellm_shadowevaljob.create = AsyncMock(return_value=_job_record()) - prisma.db.litellm_shadowevaljob.update = AsyncMock( - return_value=_job_record(stopped_at=datetime.now(timezone.utc)) - ) + prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record(token) for token in known_keys]) + async def execute_raw(sql: str, *params: object): + if "SET stopped_by" in sql: + group = [row for row in stored if row.group_id == params[0]] + counts = {row["job_id"]: row["attempt_count"] for row in prisma.attempt_rows} + sampling = any(row.stopped_at is None and counts.get(row.id, 0) < row.max_turns for row in group) + window_open = bool(group) and group[0].ends_at > datetime.now(timezone.utc) + claimable = [row for row in group if row.stopped_by is None] + if not (claimable and sampling and window_open): + return 0 + for row in claimable: + row.stopped_by = params[1] + if row.stopped_at is None: + row.stopped_at = datetime.fromisoformat(str(params[2])).replace(tzinfo=timezone.utc) + return len(claimable) + return 0 + + prisma.db.execute_raw = AsyncMock(side_effect=execute_raw) + stored = legs if isinstance(legs, list) else list(legs) + + async def find_many_legs(where=None, **_: object): + current = list(stored) + w = dict(where or {}) + if "api_key_id" in w: + wanted = w["api_key_id"]["in"] if isinstance(w["api_key_id"], dict) else [w["api_key_id"]] + current = [row for row in current if row.api_key_id in wanted] + if "direction" in w: + current = [row for row in current if row.direction == w["direction"]] + if "stopped_at" in w: + current = [row for row in current if row.stopped_at is w["stopped_at"]] + if "group_id" in w: + wanted = w["group_id"]["in"] if isinstance(w["group_id"], dict) else [w["group_id"]] + current = [row for row in current if row.group_id in wanted] + return current + + def newest_groups(rows, limit): + latest: dict = {} + for row in rows: + if row.group_id not in latest or row.created_at > latest[row.group_id]: + latest[row.group_id] = row.created_at + ordered = sorted(latest, key=lambda group_id: latest[group_id], reverse=True) + return ordered[: int(limit)] + + def leg_dict(row): + fields = ( + "id", + "group_id", + "api_key_id", + "router_name", + "direction", + "baseline_model", + "judge_model", + "shadow_percentage", + "max_turns", + "created_at", + "ends_at", + "stopped_at", + "stopped_by", + ) + return {field: getattr(row, field) for field in fields} + + prisma.db.litellm_shadowevaljob.find_many = AsyncMock(side_effect=find_many_legs) + prisma.db.litellm_shadowevaljob.create_many = AsyncMock(return_value=1) + prisma.db.litellm_shadowevaljob.update_many = AsyncMock(return_value=1) prisma.db.litellm_shadowevalattempt.find_first = AsyncMock(return_value=None) + prisma.attempt_rows = [] async def query_raw(sql: str, *params: object): + if "AS attempt_count" in sql: + return prisma.attempt_rows + if "GROUP BY group_id" in sql: + scoped = [row for row in stored if "api_key_id = $2" not in sql or row.api_key_id == params[1]] + keep = set(newest_groups(scoped, params[0])) + return [leg_dict(row) for row in stored if row.group_id in keep] if "FILTER (WHERE outcome != 'error')::int AS judged_count" in sql: return [{"judged_count": 10, "error_count": 2, "judge_spend": 0.031}] + if "SELECT job_id AS grp" in sql: + return by_leg_rows if by_leg_rows is not None else [] return agg_rows if agg_rows is not None else [] prisma.db.query_raw = AsyncMock(side_effect=query_raw) @@ -563,7 +633,7 @@ def _shadow_prisma(active_job=None, agg_rows=None) -> MagicMock: def _start_request(**overrides: object) -> StartShadowEvalRequest: payload = { - "api_key_id": "key-hash", + "api_key_ids": ("key-hash",), "router_name": "my-router", "shadow_percentage": 10.0, "judge_model": "anthropic/claude-sonnet-5", @@ -575,44 +645,55 @@ def _start_request(**overrides: object) -> StartShadowEvalRequest: @pytest.mark.asyncio -async def test_start_shadow_eval_creates_job_and_frees_expired_or_exhausted_ones(monkeypatch: pytest.MonkeyPatch): - """Expiry and turn-budget exhaustion both end sampling on their own; either must - release the key's slot in the active-job index so a new eval can start.""" +async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeypatch: pytest.MonkeyPatch): + """N keys become N sibling rows sharing group_id and identical config, written by a + single create_many so a unique-index loser rolls back the whole claim, and expiry or + budget exhaustion frees every requested key's slot first.""" import litellm.proxy.proxy_server as proxy_server prisma = _shadow_prisma() monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - response = await start_shadow_eval(_start_request(), ADMIN) + response = await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) - assert response.status == "running" - assert response.max_turns == 200 - assert response.judged_count is None - sweep_sql, sweep_key = prisma.db.execute_raw.call_args.args + sweep_sql, sweep_keys = prisma.db.execute_raw.call_args.args assert "stopped_at IS NULL" in sweep_sql - assert "ends_at <= NOW()" in sweep_sql + assert "j.ends_at <= (NOW() AT TIME ZONE 'utc')" in sweep_sql + assert "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in sweep_sql assert ">= j.max_turns" in sweep_sql - assert sweep_key == "key-hash" - create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"] - assert create_data["api_key_id"] == "key-hash" - assert create_data["created_by"] == "admin" - assert "status" not in create_data + assert "j.api_key_id = ANY($1::text[])" in sweep_sql + assert sweep_keys == ["key-hash", "key-hash-2"] + prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert [row["api_key_id"] for row in rows] == ["key-hash", "key-hash-2"] + assert len({frozenset((k, v) for k, v in row.items() if k != "api_key_id") for row in rows}) == 1 + assert len({row["group_id"] for row in rows}) == 1 + assert all(row["max_turns"] == 200 and row["created_by"] == "admin" for row in rows) + assert all("status" not in row and "id" not in row for row in rows) + assert response.job_id == rows[0]["group_id"] + assert response.status == "running" + assert response.judged_count is None + assert [(key.api_key_id, key.max_turns, key.key_alias) for key in response.keys] == [ + ("key-hash", 200, "prod-alpha"), + ("key-hash-2", 200, "prod-alpha"), + ] @pytest.mark.asyncio @pytest.mark.parametrize( - "caller,request_overrides,active,expected_status", + "caller,request_overrides,claimed,expected_status", [ - (NON_ADMIN, {}, None, 403), - (VIEWER, {}, None, 403), - (ADMIN, {"router_name": "not-a-router"}, None, 400), - (ADMIN, {"judge_model": "not/a real model!"}, None, 400), - (ADMIN, {"judge_model": "my-router"}, None, 400), - (ADMIN, {}, "active", 409), - (ADMIN, {"direction": "reverse", "baseline_model": "my-router"}, None, 400), - (ADMIN, {"direction": "reverse", "baseline_model": "not/a real model!"}, None, 400), - (ADMIN, {"direction": "reverse", "baseline_model": "openai/gpt-4o", "router_name": "not-a-router"}, None, 400), + (NON_ADMIN, {}, (), 403), + (VIEWER, {}, (), 403), + (ADMIN, {"router_name": "not-a-router"}, (), 400), + (ADMIN, {"judge_model": "not/a real model!"}, (), 400), + (ADMIN, {"judge_model": "my-router"}, (), 400), + (ADMIN, {}, ("key-hash",), 409), + (ADMIN, {"api_key_ids": ("key-hash", "key-hash-2")}, ("key-hash-2",), 409), + (ADMIN, {"direction": "reverse", "baseline_model": "my-router"}, (), 400), + (ADMIN, {"direction": "reverse", "baseline_model": "not/a real model!"}, (), 400), + (ADMIN, {"direction": "reverse", "baseline_model": "openai/gpt-4o", "router_name": "not-a-router"}, (), 400), ], ids=[ "non-admin", @@ -621,23 +702,143 @@ async def test_start_shadow_eval_creates_job_and_frees_expired_or_exhausted_ones "unresolvable-judge", "router-as-judge", "already-active", + "one-of-several-keys-already-active", "router-as-baseline", "unresolvable-baseline", "reverse-still-needs-an-auto-router", ], ) async def test_start_shadow_eval_rejections( - monkeypatch: pytest.MonkeyPatch, caller, request_overrides, active, expected_status + monkeypatch: pytest.MonkeyPatch, caller, request_overrides, claimed, expected_status ): import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma(active_job=_job_record() if active else None) + prisma = _shadow_prisma(legs=[_leg_record(id=f"leg-{key}", group_id="job-7", api_key_id=key) for key in claimed]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) with pytest.raises(HTTPException) as exc: await start_shadow_eval(_start_request(**request_overrides), caller) assert exc.value.status_code == expected_status + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_start_shadow_eval_names_the_busy_key_and_its_job(monkeypatch: pytest.MonkeyPatch): + """A key busy elsewhere blocks the whole start rather than being silently dropped from + it, and the 409 names which key and which job so the caller can stop or drop it.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(id="leg-b", group_id="job-7", api_key_id="key-hash-2")]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(api_key_ids=("key-hash", "key-hash-2")), ADMIN) + assert exc.value.status_code == 409 + assert "key-hash-2 (job job-7)" in exc.value.detail + + +@pytest.mark.asyncio +async def test_start_shadow_eval_reuses_a_key_whose_previous_job_already_stopped(monkeypatch: pytest.MonkeyPatch): + """The claim is held by unstopped legs only, matching the partial unique index. A read + that forgets that would strand every key that has ever finished a job.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(group_id="job-7", stopped_at=datetime.now(timezone.utc))]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + job = await start_shadow_eval(_start_request(), ADMIN) + + assert job.status == "running" + prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_start_shadow_eval_reverse_records_its_arms_and_holds_its_own_slot(monkeypatch: pytest.MonkeyPatch): + """The two directions ask opposite questions of the same key, so a forward job holding + the slot must not block a reverse one. The second reverse start still 409s.""" + import litellm.proxy.proxy_server as proxy_server + + legs = [_leg_record(group_id="job-fwd")] + prisma = _shadow_prisma(legs=legs) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + reverse = _start_request(direction="reverse", baseline_model="openai/gpt-4o") + response = await start_shadow_eval(reverse, ADMIN) + + assert (response.direction, response.baseline_model) == ("reverse", "openai/gpt-4o") + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert rows[0]["direction"] == "reverse" + assert rows[0]["baseline_model"] == "openai/gpt-4o" + + legs.append(_leg_record(id="leg-2", group_id="job-rev", direction="reverse")) + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(reverse, ADMIN) + assert exc.value.status_code == 409 + + +@pytest.mark.asyncio +async def test_start_shadow_eval_forward_leaves_the_baseline_column_empty(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma() + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + await start_shadow_eval(_start_request(), ADMIN) + + rows = prisma.db.litellm_shadowevaljob.create_many.call_args.kwargs["data"] + assert rows[0]["direction"] == "forward" + assert rows[0]["baseline_model"] is None + + +@pytest.mark.asyncio +async def test_start_shadow_eval_rejects_keys_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch): + """A typo'd api_key_id would otherwise create a leg no traffic can ever match. Every + unknown key is named at once, so a caller passing several fixes them in one round.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(known_keys=("key-hash",)) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(api_key_ids=("key-hash", "typo-a", "typo-b")), ADMIN) + assert exc.value.status_code == 400 + assert "typo-a, typo-b" in exc.value.detail + assert "key-hash," not in exc.value.detail + prisma.db.litellm_shadowevaljob.create_many.assert_not_called() + + +def test_start_shadow_eval_request_dedupes_and_bounds_the_key_set(): + """A key named twice would collide with itself on the one-active-per-key index, a job + scoping no key samples nothing, and the key-count cap bounds every downstream read.""" + assert _start_request(api_key_ids=("a", "b", "a")).api_key_ids == ("a", "b") + assert len(_start_request(api_key_ids=tuple(f"k{i}" for i in range(100))).api_key_ids) == 100 + with pytest.raises(ValidationError): + _start_request(api_key_ids=()) + with pytest.raises(ValidationError): + _start_request(api_key_ids=tuple(f"k{i}" for i in range(101))) + + +@pytest.mark.asyncio +async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + from prisma.errors import UniqueViolationError + + prisma = _shadow_prisma() + prisma.db.litellm_shadowevaljob.create_many = AsyncMock( + side_effect=UniqueViolationError(MagicMock(message="unique constraint")) + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) + + with pytest.raises(HTTPException) as exc: + await start_shadow_eval(_start_request(), ADMIN) + assert exc.value.status_code == 409 @pytest.mark.parametrize( @@ -657,97 +858,25 @@ def test_start_request_pins_baseline_model_to_reverse(overrides): @pytest.mark.asyncio -async def test_start_shadow_eval_reverse_records_its_arms_and_holds_its_own_slot(monkeypatch: pytest.MonkeyPatch): - """The two directions ask opposite questions of the same key, so a forward job holding - the slot must not block a reverse one. The second reverse start still 409s.""" - import litellm.proxy.proxy_server as proxy_server - - prisma = _shadow_prisma() - active = {"forward": _job_record()} - prisma.db.litellm_shadowevaljob.find_first = AsyncMock( - side_effect=lambda where, **_: active.get(str(where.get("direction"))) - ) - prisma.db.litellm_shadowevaljob.create = AsyncMock( - return_value=_job_record(direction="reverse", baseline_model="openai/gpt-4o") - ) - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - reverse = _start_request(direction="reverse", baseline_model="openai/gpt-4o") - response = await start_shadow_eval(reverse, ADMIN) - - assert (response.direction, response.baseline_model) == ("reverse", "openai/gpt-4o") - create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"] - assert create_data["direction"] == "reverse" - assert create_data["baseline_model"] == "openai/gpt-4o" - - active["reverse"] = _job_record(id="job-2", direction="reverse") - with pytest.raises(HTTPException) as exc: - await start_shadow_eval(reverse, ADMIN) - assert exc.value.status_code == 409 - - -@pytest.mark.asyncio -async def test_start_shadow_eval_forward_leaves_the_baseline_column_empty(monkeypatch: pytest.MonkeyPatch): - import litellm.proxy.proxy_server as proxy_server - - prisma = _shadow_prisma() - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - await start_shadow_eval(_start_request(), ADMIN) - - create_data = prisma.db.litellm_shadowevaljob.create.call_args.kwargs["data"] - assert create_data["direction"] == "forward" - assert create_data["baseline_model"] is None - - -@pytest.mark.asyncio -async def test_start_shadow_eval_rejects_a_key_this_proxy_does_not_know(monkeypatch: pytest.MonkeyPatch): - """A typo'd api_key_id would otherwise create a job no traffic can ever match.""" - import litellm.proxy.proxy_server as proxy_server - - prisma = _shadow_prisma() - prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - with pytest.raises(HTTPException) as exc: - await start_shadow_eval(_start_request(), ADMIN) - assert exc.value.status_code == 400 - assert "not a key on this proxy" in exc.value.detail - - -@pytest.mark.asyncio -async def test_start_shadow_eval_concurrent_unique_violation_is_a_409(monkeypatch: pytest.MonkeyPatch): - import litellm.proxy.proxy_server as proxy_server - from prisma.errors import UniqueViolationError - - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.create = AsyncMock( - side_effect=UniqueViolationError(MagicMock(message="unique constraint")) - ) - monkeypatch.setattr(proxy_server, "prisma_client", prisma) - monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) - - with pytest.raises(HTTPException) as exc: - await start_shadow_eval(_start_request(), ADMIN) - assert exc.value.status_code == 409 - - -@pytest.mark.asyncio -async def test_get_shadow_eval_job_derives_counts_spend_and_stratified_results(monkeypatch: pytest.MonkeyPatch): +async def test_get_shadow_eval_job_pools_counts_and_slices_results_per_key(monkeypatch: pytest.MonkeyPatch): + """One read answers for every leg: totals and stratifications aggregate over the + group's leg ids, and the by-key slice maps each leg id back to its key hash.""" import litellm.proxy.proxy_server as proxy_server tier_rows = [ {"grp": "SIMPLE", "turn_count": 8, "real_wins": 2, "shadow_wins": 4, "ties": 2, "avg_confidence": 0.8}, {"grp": "REASONING", "turn_count": 2, "real_wins": 2, "shadow_wins": 0, "ties": 0, "avg_confidence": 0.9}, ] - prisma = _shadow_prisma(agg_rows=tier_rows) - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) - prisma.db.litellm_shadowevalattempt.find_first = AsyncMock( - return_value=MagicMock(error="judge call failed: boom") + leg_rows = [ + {"grp": "leg-1", "turn_count": 6, "real_wins": 1, "shadow_wins": 4, "ties": 1, "avg_confidence": 0.7}, + {"grp": "leg-2", "turn_count": 4, "real_wins": 3, "shadow_wins": 0, "ties": 1, "avg_confidence": 0.6}, + ] + prisma = _shadow_prisma( + legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=50)], + agg_rows=tier_rows, + by_leg_rows=leg_rows, ) + prisma.db.litellm_shadowevalattempt.find_first = AsyncMock(return_value=MagicMock(error="judge call failed: boom")) monkeypatch.setattr(proxy_server, "prisma_client", prisma) response = await get_shadow_eval_job("job-1", VIEWER) @@ -762,6 +891,13 @@ async def test_get_shadow_eval_job_derives_counts_spend_and_stratified_results(m assert response.results.by_tier[0].shadow_win_rate_pct == 50.0 assert response.results.overall_shadow_win_rate_pct == 40.0 assert response.results.overall_tie_rate_pct == 20.0 + assert [(s.group, s.turn_count) for s in response.results.by_key] == [("key-hash", 6), ("key-hash-2", 4)] + assert response.results.by_key[0].shadow_win_rate_pct == 66.7 + assert [(key.api_key_id, key.max_turns) for key in response.keys] == [("key-hash", 200), ("key-hash-2", 50)] + totals_args = [call.args for call in prisma.db.query_raw.await_args_list if "judged_count" in call.args[0]] + assert totals_args == [(totals_args[0][0], ["leg-1", "leg-2"])] + error_where = prisma.db.litellm_shadowevalattempt.find_first.call_args.kwargs["where"] + assert error_where == {"job_id": {"in": ["leg-1", "leg-2"]}, "outcome": "error"} @pytest.mark.asyncio @@ -780,79 +916,326 @@ async def test_get_shadow_eval_job_404s_and_gates_on_role(monkeypatch: pytest.Mo @pytest.mark.asyncio -async def test_list_shadow_eval_jobs_returns_derived_status_without_aggregates(monkeypatch: pytest.MonkeyPatch): +async def test_list_shadow_eval_jobs_collapses_legs_into_jobs_newest_first(monkeypatch: pytest.MonkeyPatch): + """A job over two keys is one list entry with both keys, not two entries, and a job + whose keys all stopped reads stopped while a half-stopped one still runs.""" import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.find_many = AsyncMock( - return_value=[ - _job_record(), - _job_record(id="job-2", ends_at=datetime.now(timezone.utc) - timedelta(days=1)), - _job_record(id="job-3", stopped_at=datetime.now(timezone.utc)), + stamp = datetime.now(timezone.utc) + prisma = _shadow_prisma( + legs=[ + _leg_record(created_at=datetime(2026, 8, 13, tzinfo=timezone.utc)), + _leg_record( + id="leg-2", + api_key_id="key-hash-2", + stopped_at=stamp, + created_at=datetime(2026, 8, 13, tzinfo=timezone.utc), + ), + _leg_record( + id="leg-3", + group_id="job-2", + stopped_at=stamp, + created_at=datetime(2026, 8, 12, tzinfo=timezone.utc), + ), + _leg_record( + id="leg-4", + group_id="job-3", + ends_at=datetime.now(timezone.utc) - timedelta(days=1), + created_at=datetime(2026, 8, 11, tzinfo=timezone.utc), + ), ] ) monkeypatch.setattr(proxy_server, "prisma_client", prisma) jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) - assert [job.status for job in jobs] == ["running", "completed", "stopped"] - swept = ShadowEvalJobResponse.model_validate( - _job_record( - id="job-4", - ends_at=datetime.now(timezone.utc) - timedelta(days=1), - stopped_at=datetime.now(timezone.utc), - ), - from_attributes=True, - ) - assert swept.status == "completed" + assert [(job.job_id, job.status) for job in jobs] == [ + ("job-1", "running"), + ("job-2", "stopped"), + ("job-3", "completed"), + ] + assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"] assert all(job.judged_count is None and job.results is None for job in jobs) - assert prisma.db.query_raw.await_count == 0 + legs_sql, legs_limit = prisma.db.query_raw.await_args_list[0].args + assert "GROUP BY group_id ORDER BY MAX(created_at) DESC LIMIT $1::int" in legs_sql + assert legs_limit == 50 + counts_sql, _ = prisma.db.query_raw.await_args_list[1].args + assert "AS attempt_count" in counts_sql + assert "j.stopped_at IS NULL OR a.created_at <= j.stopped_at" in counts_sql + assert prisma.db.query_raw.await_count == 2 + prisma.db.litellm_shadowevaljob.find_many.assert_not_called() @pytest.mark.asyncio -async def test_shadow_eval_responses_name_the_shadowed_key(monkeypatch: pytest.MonkeyPatch): +async def test_list_shadow_eval_jobs_filters_to_jobs_containing_the_key(monkeypatch: pytest.MonkeyPatch): + """The filter matches a key anywhere in a job's key set and still returns the whole + job, sibling keys included.""" import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.find_many = AsyncMock( - return_value=[_job_record(), _job_record(id="job-2", api_key_id="deleted-key-hash")] + prisma = _shadow_prisma( + legs=[ + _leg_record(), + _leg_record(id="leg-2", api_key_id="key-hash-2"), + _leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash-2"), + _leg_record(id="leg-4", group_id="job-3"), + ] + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id="key-hash-2", limit=50) + + assert [job.job_id for job in jobs] == ["job-1", "job-2"] + assert [key.api_key_id for key in jobs[0].keys] == ["key-hash", "key-hash-2"] + + +@pytest.mark.parametrize( + ("stopped_flags", "days_left", "expected"), + [ + ((False, False), 7, "running"), + ((True, False), 7, "running"), + ((True, True), 7, "stopped"), + ((True, True), -1, "completed"), + ((False, False), -1, "completed"), + ], +) +@pytest.mark.asyncio +async def test_job_status_runs_until_every_key_stops_and_completed_outranks_stopped( + monkeypatch: pytest.MonkeyPatch, stopped_flags: tuple[bool, ...], days_left: int, expected: str +): + import litellm.proxy.proxy_server as proxy_server + + stamp = datetime.now(timezone.utc) + prisma = _shadow_prisma( + legs=[ + _leg_record( + id=f"leg-{index}", + api_key_id=f"key-{index}", + stopped_at=stamp if stopped else None, + ends_at=datetime.now(timezone.utc) + timedelta(days=days_left), + ) + for index, stopped in enumerate(stopped_flags) + ] + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + + assert [job.status for job in jobs] == [expected] + + +@pytest.mark.asyncio +async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch: pytest.MonkeyPatch): + """A job whose keys all exhausted their turn budgets stopped sampling on its own, so + it must read completed on the very next list, before any sweep stamps its legs; one + key under budget keeps the whole job running. An operator starting an unrelated eval + must never look like it terminated a finished one.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma( + legs=[ + _leg_record(max_turns=5), + _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=5), + _leg_record(id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=5), + _leg_record(id="leg-4", group_id="job-2", api_key_id="key-hash-2", max_turns=5), + ] + ) + prisma.attempt_rows = [ + {"job_id": "leg-1", "attempt_count": 5}, + {"job_id": "leg-2", "attempt_count": 6}, + {"job_id": "leg-3", "attempt_count": 5}, + {"job_id": "leg-4", "attempt_count": 3}, + ] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + + by_id = {job.job_id: job for job in jobs} + assert by_id["job-1"].status == "completed" + assert all(key.stopped_at is None for key in by_id["job-1"].keys) + assert by_id["job-2"].status == "running" + assert {key.api_key_id: key.attempt_count for key in by_id["job-2"].keys} == {"key-hash": 5, "key-hash-2": 3} + + +@pytest.mark.asyncio +async def test_recorded_operator_stop_outranks_budget_arithmetic(monkeypatch: pytest.MonkeyPatch): + """A detached attempt can land around the stop and push the raw count past the + budget; the recorded stopped_by must keep the job reading stopped regardless.""" + import litellm.proxy.proxy_server as proxy_server + + stamp = datetime.now(timezone.utc) + prisma = _shadow_prisma(legs=[_leg_record(max_turns=5, stopped_at=stamp, stopped_by="admin")]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + assert jobs[0].status == "stopped" + assert jobs[0].stopped_by == "admin" + + detail = await get_shadow_eval_job("job-1", VIEWER) + assert detail.status == "stopped" + + +@pytest.mark.asyncio +async def test_backfilled_legacy_stop_never_reads_as_completion(monkeypatch: pytest.MonkeyPatch): + """Jobs stopped before stopped_by existed are backfilled with 'unknown' by the + migration, so even one whose stray attempts crossed the budget stays stopped.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma( + legs=[_leg_record(max_turns=5, stopped_at=datetime.now(timezone.utc), stopped_by="unknown")] + ) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + assert jobs[0].status == "stopped" + + +def test_stopped_by_migration_backfills_every_job_that_displayed_stopped(): + """The migration must close the pre-column population: without the backfill, a + legacy stop whose stray attempts crossed the budget would read completed.""" + import litellm_proxy_extras + + sql = ( + Path(litellm_proxy_extras.__file__).parent + / "migrations" + / "20260818224500_add_shadow_eval_stopped_by" + / "migration.sql" + ).read_text() + assert 'ADD COLUMN "stopped_by" TEXT' in sql + assert "SET stopped_by = 'unknown'" in sql + assert "WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc')" in sql + + +@pytest.mark.asyncio +async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(max_turns=3)]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 3}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + with pytest.raises(HTTPException) as exhausted: + await stop_shadow_eval_job("job-1", ADMIN) + assert exhausted.value.status_code == 400 + assert "completed" in exhausted.value.detail + prisma.db.litellm_shadowevaljob.update_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma( + legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="deleted-key-hash")], + known_keys=("key-hash", "key-hash-2"), ) monkeypatch.setattr(proxy_server, "llm_router", _shadow_router()) monkeypatch.setattr(proxy_server, "prisma_client", prisma) - started = await start_shadow_eval(_start_request(), ADMIN) - assert (started.key_alias, started.key_name) == ("prod-alpha", "sk-...lpha") - jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) - assert [(job.key_alias, job.key_name) for job in jobs] == [("prod-alpha", "sk-...lpha"), (None, None)] + assert [(key.key_alias, key.key_name) for key in jobs[0].keys] == [ + (None, None), + ("prod-alpha", "sk-...lpha"), + ] batched_where = prisma.db.litellm_verificationtoken.find_many.call_args.kwargs["where"] assert batched_where == {"token": {"in": ["deleted-key-hash", "key-hash"]}} - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) detail = await get_shadow_eval_job("job-1", VIEWER) - assert detail.key_alias == "prod-alpha" + assert [key.key_alias for key in detail.keys] == [None, "prod-alpha"] @pytest.mark.asyncio -async def test_stop_shadow_eval_sets_stopped_at_and_rejects_non_running(monkeypatch: pytest.MonkeyPatch): +async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_running( + monkeypatch: pytest.MonkeyPatch, +): + """One stop ends sampling for the whole job, while a leg that already stopped on its + own budget keeps the stopped_at it earned.""" import litellm.proxy.proxy_server as proxy_server - prisma = _shadow_prisma() - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock(return_value=_job_record()) + earned = datetime.now(timezone.utc) - timedelta(hours=1) + prisma = _shadow_prisma(legs=[_leg_record(), _leg_record(id="leg-2", api_key_id="key-hash-2", stopped_at=earned)]) monkeypatch.setattr(proxy_server, "prisma_client", prisma) stopped = await stop_shadow_eval_job("job-1", ADMIN) - assert stopped.status == "stopped" - update = prisma.db.litellm_shadowevaljob.update.call_args.kwargs - assert set(update["data"]) == {"stopped_at"} - prisma.db.litellm_shadowevaljob.find_unique = AsyncMock( - return_value=_job_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1)) - ) + assert stopped.status == "stopped" + assert stopped.stopped_by == "admin" + stop_sql, stop_group, stop_operator, stop_stamp = prisma.db.execute_raw.call_args.args + assert "SET stopped_by = $2, stopped_at = COALESCE(stopped_at, $3::timestamp)" in stop_sql + assert "WHERE group_id = $1 AND stopped_by IS NULL" in stop_sql + assert "ends_at > (NOW() AT TIME ZONE 'utc')" in stop_sql + assert ") < k.max_turns" in stop_sql + assert (stop_group, stop_operator) == ("job-1", "admin") + assert datetime.fromisoformat(stop_stamp).tzinfo is None + assert prisma.db.execute_raw.await_count == 1 + prisma.db.litellm_shadowevaljob.update_many.assert_not_called() + by_key = {key.api_key_id: key.stopped_at for key in stopped.keys} + assert by_key["key-hash-2"] == earned + assert by_key["key-hash"] is not None and by_key["key-hash"] != earned + + done_leg = _leg_record(ends_at=datetime.now(timezone.utc) - timedelta(days=1)) + prisma_done = _shadow_prisma(legs=[done_leg]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_done) with pytest.raises(HTTPException) as exc: await stop_shadow_eval_job("job-1", ADMIN) assert exc.value.status_code == 400 + assert "already completed" in exc.value.detail + assert done_leg.stopped_by is None with pytest.raises(HTTPException) as forbidden: await stop_shadow_eval_job("job-1", VIEWER) assert forbidden.value.status_code == 403 + + +def test_every_shadow_eval_sql_constant_speaks_naive_utc(): + """The tables store naive UTC wall time (prisma's convention), so SQL-side time must be + NOW() AT TIME ZONE 'utc' and python-side params must cast ::timestamp; a bare NOW() or a + timestamptz cast writes session-local wall time into the naive column and skews every + comparison against prisma-written stamps.""" + import litellm.proxy.management_endpoints.auto_router_endpoints as module + + sql_constants = {name: value for name, value in vars(module).items() if name.endswith("_SQL")} + assert sql_constants + for name, sql in sql_constants.items(): + assert "::timestamptz" not in sql, name + for occurrence in sql.split("NOW()")[1:]: + assert occurrence.startswith(" AT TIME ZONE 'utc'"), name + + +@pytest.mark.asyncio +async def test_a_stop_racing_the_last_budgeted_attempt_reports_completed_not_stopped( + monkeypatch: pytest.MonkeyPatch, +): + """The statement claims the job only while a leg still samples, so a stop landing in + the same instant the budget spends records nothing and the job keeps reading + completed; stamping it would misreport a self-ended job as operator-stopped forever.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(max_turns=2)]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 2}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + with pytest.raises(HTTPException) as exc: + await stop_shadow_eval_job("job-1", ADMIN) + assert exc.value.status_code == 400 + assert "already completed" in exc.value.detail + assert prisma.db.litellm_shadowevaljob.find_many.await_args.kwargs["where"] == {"group_id": "job-1"} + + +@pytest.mark.asyncio +async def test_two_racing_stops_produce_exactly_one_winner(monkeypatch: pytest.MonkeyPatch): + """The statement's stopped_by IS NULL predicate lets only one racer claim rows; the + loser reads the stamped state and gets the same answer a late caller gets.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record()]) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + first = await stop_shadow_eval_job("job-1", ADMIN) + assert first.status == "stopped" + + with pytest.raises(HTTPException) as exc: + await stop_shadow_eval_job("job-1", ADMIN) + assert exc.value.status_code == 400 + assert "already stopped" in exc.value.detail diff --git a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py index 736fc13d137..f41756d2b87 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py +++ b/tests/test_litellm/proxy/spend_tracking/test_ptu_flat_cost_rollup.py @@ -1,5 +1,6 @@ """Tests for the per-model PTU flat-cost daily rollup.""" +import json import types from datetime import date, datetime, timedelta, timezone from unittest.mock import AsyncMock, MagicMock @@ -373,6 +374,182 @@ def test_parse_ptu_model_rejects_an_inverted_window(): assert parsed is None +def _router_entry(model_id="dep-1", model_name="gpt-4o-ptu", model_info=None, with_start=True): + """A deployment as the router stores one: a plain dict whose id lives in model_info.""" + info = dict(model_info or {}) + if ( + with_start + and info.get("ptu_count") is not None + and info.get("cost_per_ptu_per_hour") is not None + and "ptu_effective_from" not in info + ): + info["ptu_effective_from"] = _DEFAULT_PTU_START + if model_id is not None: + info["id"] = model_id + return { + "model_name": model_name, + "litellm_params": {"model": "azure/gpt-4o"}, + "model_info": info, + } + + +# A team-scoped deployment is stored under a synthetic routing key with the operator's name +# in team_public_model_name, while the same deployment in config.yaml carries the operator's +# name directly. Both must resolve to the same PTUModel or the two sources bill differently. +_PARITY_CASES = ( + ( + "an iso string start against the datetime pydantic coerces it to", + {"ptu_effective_from": "2026-07-30T23:00:00Z"}, + {"ptu_effective_from": datetime(2026, 7, 30, 23, 0)}, + datetime(2026, 7, 30, 23, 0, tzinfo=timezone.utc), + None, + ), + ( + "an open window, stored as null and dropped by exclude_none", + {"ptu_effective_from": "2026-07-01T00:00:00Z", "ptu_effective_to": None}, + {"ptu_effective_from": datetime(2026, 7, 1, 0, 0)}, + datetime(2026, 7, 1, tzinfo=timezone.utc), + None, + ), + ( + "a closed window", + {"ptu_effective_from": "2026-07-01T00:00:00Z", "ptu_effective_to": "2026-07-31T00:00:00Z"}, + { + "ptu_effective_from": datetime(2026, 7, 1, 0, 0), + "ptu_effective_to": datetime(2026, 7, 31, 0, 0), + }, + datetime(2026, 7, 1, tzinfo=timezone.utc), + datetime(2026, 7, 31, tzinfo=timezone.utc), + ), +) + + +@pytest.mark.parametrize( + "db_window, router_window, expected_from, expected_to", + [case[1:] for case in _PARITY_CASES], + ids=[case[0] for case in _PARITY_CASES], +) +def test_a_deployment_parses_identically_from_the_db_and_from_config_yaml( + db_window, router_window, expected_from, expected_to +): + # The contract the router union is built on. If these ever diverge, a PTU deployment + # declared in config.yaml is billed differently from the identical one in the database. + from_db = _parse_ptu_model( + _model_row( + model_id="dep-1", + model_name="model_name_t_9f3c2b", + model_info={**_VALID_PTU, **db_window, "team_public_model_name": "gpt-4o-ptu"}, + with_start=False, + ) + ) + from_router = _parse_ptu_model( + ptu_rollup._router_deployment( + _router_entry(model_id="dep-1", model_info={**_VALID_PTU, **router_window}, with_start=False) + ) + ) + expected = PTUModel( + model_id="dep-1", + model_name="gpt-4o-ptu", + team_id="t", + ptu_count=5, + cost_per_ptu_per_hour=2.0, + effective_from=expected_from, + effective_to=expected_to, + ) + assert from_db == from_router == expected + + +@pytest.mark.parametrize("model_id", [None, "", 12345], ids=["absent", "blank", "not a string"]) +def test_a_router_deployment_without_a_usable_id_is_dropped(model_id): + # model_id keys the sentinel row, so an unusable one would file every such deployment + # in a team onto one row and bill for a single reservation. + entry = _router_entry(model_id=None, model_info=dict(_VALID_PTU)) + if model_id is not None: + entry["model_info"]["id"] = model_id + assert ptu_rollup._router_deployment(entry) is None + + +def test_a_router_deployment_keeps_the_name_the_operator_wrote(): + priced = _parse_ptu_model( + ptu_rollup._router_deployment(_router_entry(model_name="gpt-4o-ptu", model_info=dict(_VALID_PTU))) + ) + assert priced is not None + assert priced.model_name == "gpt-4o-ptu" + + +def test_a_team_alias_on_a_router_deployment_still_wins_over_the_routing_name(): + # config.yaml can carry team_public_model_name to expose a team-facing alias, and the + # charge has to file under the name that team calls rather than the routing one. + priced = _parse_ptu_model( + ptu_rollup._router_deployment( + _router_entry( + model_name="routing-name", + model_info={**_VALID_PTU, "team_public_model_name": "public-alias"}, + ) + ) + ) + assert priced is not None + assert priced.model_name == "public-alias" + + +def test_a_router_deployment_with_a_stringified_model_info_still_decodes(): + entry = _router_entry(model_info=dict(_VALID_PTU)) + entry["model_info"] = json.dumps(entry["model_info"]) + parsed = _parse_ptu_model(ptu_rollup._router_deployment(entry)) + assert parsed is not None + assert parsed.model_id == "dep-1" + + +@pytest.mark.parametrize( + "model_info", + [None, "not json", 42, "[1, 2, 3]", '"a string"', "42", "true", "null"], + ids=[ + "absent", + "unparseable", + "not a string or dict", + "json array", + "json string", + "json number", + "json bool", + "json null", + ], +) +def test_a_router_deployment_without_usable_model_info_is_dropped(model_info): + # Valid JSON that is not an object decodes to a list or a scalar, and reading fields + # off one raises rather than dropping the single bad deployment the rollup expects + assert ptu_rollup._router_deployment({"model_name": "x", "model_info": model_info}) is None + + +@pytest.mark.parametrize( + "model_info", + ["[1, 2, 3]", '"a string"', "42", "true", "null"], + ids=["json array", "json string", "json number", "json bool", "json null"], +) +def test_a_db_row_holding_non_object_json_is_dropped(model_info): + assert _parse_ptu_model(_model_row(model_info=model_info, with_start=False)) is None + + +def test_a_router_deployment_does_not_alias_the_routers_own_model_info(): + # The rollup runs on a cron while requests are in flight, and the router rewrites + # model_info in place, so a held record must neither observe nor cause those writes. + live = {"id": "dep-1", **_VALID_PTU} + record = ptu_rollup._router_deployment({"model_name": "x", "model_info": live}) + assert record is not None + + live["ptu_count"] = 999 + assert record.model_info["ptu_count"] == _VALID_PTU["ptu_count"] + + with pytest.raises(TypeError): + record.model_info["ptu_count"] = 1 + + +def test_a_raw_router_dict_is_not_a_deployment_record(): + # The parser reads attributes, so a router dict passed straight to it returns None + # instead of raising. Skipping the factory would silently drop every config.yaml + # deployment while leaving the suite green. + assert _parse_ptu_model(_router_entry(model_info=dict(_VALID_PTU))) is None + + @pytest.mark.asyncio async def test_rollup_returns_empty_when_prisma_client_is_none(): result = await run_ptu_flat_cost_rollup(None, target_date=DAY) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index c471355db68..09e602e269b 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -336,9 +336,6 @@ "max-params": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 3 } @@ -1302,9 +1299,6 @@ "local/filename-pascal-case": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 1 } @@ -1410,11 +1404,6 @@ "count": 2 } }, - "src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx": { "no-nested-ternary": { "count": 2 @@ -1459,9 +1448,6 @@ "local/no-complex-jsx-arrow": { "count": 1 }, - "no-restricted-imports": { - "count": 1 - }, "react-hooks/set-state-in-effect": { "count": 2 } @@ -1476,11 +1462,6 @@ "count": 1 } }, - "src/app/onboarding/OnboardingFormBody.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/AIHub/ModelHubTable.test.tsx": { "max-params": { "count": 1 @@ -2004,11 +1985,6 @@ "count": 1 } }, - "src/components/common_components/PassThroughGuardrailsSection.tsx": { - "no-restricted-imports": { - "count": 1 - } - }, "src/components/common_components/RateLimitTypeFormItem.test.tsx": { "no-restricted-imports": { "count": 1 diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 9407ff25a1f..ac13a12620d 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -9,8 +9,11 @@ "start": "next start", "lint": "eslint .", "test": "vitest", + "test:unit": "vitest run --project unit", + "test:component": "vitest run --project component", + "test:integration": "vitest run --project integration", "test:dot": "vitest --reporter=dot", - "test:types": "vitest --run --typecheck.only", + "test:types": "vitest run --project types", "test:watch": "vitest -w", "test:coverage": "vitest run --coverage", "format": "prettier --write .", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx index 48eaa62e37b..e9d7190abf5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/admin-panel/_components/AdminPanel.tsx @@ -7,8 +7,8 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; -import { Alert as AntdAlert, Modal, Space, Tabs, Typography } from "antd"; -import { Info } from "lucide-react"; +import { Modal, Space, Tabs, Typography } from "antd"; +import { Info, TriangleAlert } from "lucide-react"; import React, { useEffect, useState } from "react"; import NewBadge from "@/components/common_components/NewBadge"; import { useBaseUrl } from "@/components/constants"; @@ -223,12 +223,14 @@ const AdminPanel: React.FC = ({ proxySettings }) => { <> ✨ Security Settings - + + + SSO Configuration Deprecated + + Editing SSO Settings on this page is deprecated and will be removed in a future version. Please use the + SSO Settings tab for SSO configuration. + +
= ({ proxySettings }) => { Langchain Py - + - + - + = ({ accessToken }) => {
- +
{selectedBudget && ( @@ -134,7 +134,7 @@ const BudgetPanel: React.FC = ({ accessToken }) => { />
- +

How to use budget id

@@ -149,13 +149,13 @@ const BudgetPanel: React.FC = ({ accessToken }) => { Test it (OpenAI SDK) - + {CREATE_END_USER_CURL_COMMAND} - + {CHAT_COMPLETIONS_CURL_COMMAND} - + {OPENAI_SDK_PYTHON_CODE} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx index 62c456b1b38..cafa7a2ce37 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx @@ -140,18 +140,18 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole return ( -
- - +
+ + Cache Analytics - + Cache Health - + Cache Settings - + Coordination Redis @@ -164,7 +164,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole
- +

@@ -311,7 +311,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole - + = ({ accessToken, token, userRole /> - + - + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_health.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_health.tsx index 0f52bed874d..221a17f911d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_health.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_health.tsx @@ -164,7 +164,7 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => { - +

{response?.status === "healthy" ? ( @@ -228,7 +228,7 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => {
- + 
               {(() => {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx
index 7bb550f729e..b307e3d0f2a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx
@@ -77,7 +77,15 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({
   baseline_model: null,
   judge_model: "anthropic/claude-sonnet-5",
   shadow_percentage: 10,
-  max_turns: 200,
+  keys: [
+    {
+      api_key_id: "hashed-key-abc",
+      max_turns: 200,
+      stopped_at: null,
+      key_alias: "prod-alpha",
+      key_name: "sk-...alpha",
+    },
+  ],
   judged_count: 42,
   error_count: 1,
   judge_spend: 3.21,
@@ -110,19 +118,28 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({
         avg_judge_confidence: 0.8,
       },
     ],
+    by_key: [],
     overall_shadow_win_rate_pct: 48.0,
     overall_tie_rate_pct: 22.0,
   },
   created_at: "2026-08-07T00:00:00Z",
   ends_at: "2026-09-07T00:00:00Z",
-  stopped_at: null,
-  api_key_id: "hashed-key-abc",
-  key_alias: "prod-alpha",
-  key_name: "sk-...alpha",
   last_error: null,
   ...overrides,
 });
 
+const keyEntry = (
+  api_key_id: string,
+  overrides: Partial = {},
+): ShadowEvalJob["keys"][number] => ({
+  api_key_id,
+  max_turns: 200,
+  stopped_at: null,
+  key_alias: null,
+  key_name: null,
+  ...overrides,
+});
+
 const mockHooks = ({
   jobs = [],
   detailsById = {},
@@ -199,8 +216,8 @@ describe("ShadowEvalSection", () => {
   it("gives every active job its own card with a stop button, with the form still offered", () => {
     mockHooks({
       jobs: [
-        job({ job_id: "job-a", status: "running", api_key_id: "key-a" }),
-        job({ job_id: "job-b", status: "running", api_key_id: "key-b" }),
+        job({ job_id: "job-a", status: "running", keys: [keyEntry("key-a")] }),
+        job({ job_id: "job-b", status: "running", keys: [keyEntry("key-b")] }),
       ],
     });
     render();
@@ -342,7 +359,7 @@ describe("ShadowEvalSection", () => {
     expect(container).toBeEmptyDOMElement();
   });
 
-  it("keeps the start button disabled until key, router, and judge model are picked, then submits them", async () => {
+  it("keeps the start button disabled until key, router, and judge model are picked, then submits the key as a list", async () => {
     const user = userEvent.setup();
     const { start } = mockHooks({});
     render();
@@ -361,7 +378,7 @@ describe("ShadowEvalSection", () => {
     await user.click(screen.getByText("Start shadow eval"));
 
     const expectedBody = {
-      api_key_id: "hash-alpha",
+      api_key_ids: ["hash-alpha"],
       router_name: "gpt-auto",
       direction: "forward",
       shadow_percentage: 10,
@@ -396,7 +413,7 @@ describe("ShadowEvalSection", () => {
     await user.click(screen.getByText("Start shadow eval"));
 
     const expectedBody = {
-      api_key_id: "hash-alpha",
+      api_key_ids: ["hash-alpha"],
       router_name: "gpt-auto",
       direction: "reverse",
       baseline_model: "prod-claude",
@@ -429,9 +446,9 @@ describe("ShadowEvalSection", () => {
   });
 
   it("labels the shadowed key by alias, then masked name, then truncated hash", () => {
-    expect(shadowedKeyLabel(job())).toBe("prod-alpha");
-    expect(shadowedKeyLabel(job({ key_alias: null }))).toBe("sk-...alpha");
-    expect(shadowedKeyLabel(job({ key_alias: null, key_name: null }))).toBe("hashed-key…");
+    expect(shadowedKeyLabel(job().keys[0])).toBe("prod-alpha");
+    expect(shadowedKeyLabel(keyEntry("hashed-key-abc", { key_name: "sk-...alpha" }))).toBe("sk-...alpha");
+    expect(shadowedKeyLabel(keyEntry("hashed-key-abc"))).toBe("hashed-key…");
   });
 
   it("keeps an older job's verdicts reachable through the previous evaluations list", async () => {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx
index 005636615b1..6d240a84c98 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx
@@ -24,6 +24,7 @@ import {
   useStartShadowEval,
   useStopShadowEval,
   type ShadowEvalJob,
+  type ShadowEvalJobKey,
   type ShadowEvalSlice,
 } from "./useShadowEval";
 
@@ -50,19 +51,24 @@ const routerMatchedOrBeatPct = (
     ? 100 - results.overall_shadow_win_rate_pct
     : results.overall_shadow_win_rate_pct + results.overall_tie_rate_pct;
 
-export const shadowedKeyLabel = (job: ShadowEvalJob): string =>
-  job.key_alias || job.key_name || `${job.api_key_id.slice(0, 10)}…`;
+export const shadowedKeyLabel = (key: ShadowEvalJobKey): string =>
+  key.key_alias || key.key_name || `${key.api_key_id.slice(0, 10)}…`;
+
+const shadowedKeysLabel = (job: ShadowEvalJob): string =>
+  job.keys.length === 1 ? shadowedKeyLabel(job.keys[0]) : `${job.keys.length} keys`;
+
+const totalBudget = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + key.max_turns, 0);
 
 const jobHeadline = (job: ShadowEvalJob): React.ReactNode =>
   job.direction === "reverse" ? (
     <>
       Comparing {job.router_name} to{" "}
       {job.baseline_model} on {job.shadow_percentage}% of{" "}
-      {shadowedKeyLabel(job)} traffic
+      {shadowedKeysLabel(job)} traffic
     
   ) : (
     <>
-      Shadowing {job.shadow_percentage}% of {shadowedKeyLabel(job)} traffic
+      Shadowing {job.shadow_percentage}% of {shadowedKeysLabel(job)} traffic
       via {job.router_name}
     
   );
@@ -178,7 +184,8 @@ const emptyResultsText = (job: ShadowEvalJob, resultsError: boolean): string =>
 
 const ResultsBody: React.FC<{ job: ShadowEvalJob; resultsError?: boolean }> = ({ job, resultsError = false }) => {
   const results = job.results;
-  if (!results || (results.by_tier.length === 0 && results.by_current_model.length === 0)) {
+  const stratifications = results ? [results.by_tier, results.by_current_model, results.by_key] : [];
+  if (!results || stratifications.every((slices) => slices.length === 0)) {
     return 
{emptyResultsText(job, resultsError)}
;
   }
   return (
@@ -224,7 +231,7 @@ const JobResults: React.FC<{
           

             
{jobHeadline(job)}

             

-              {(job.judged_count ?? 0).toLocaleString()} of {job.max_turns.toLocaleString()} turns judged ·{" "}
+              {(job.judged_count ?? 0).toLocaleString()} of {totalBudget(job).toLocaleString()} turns judged ·{" "}
               {(job.error_count ?? 0).toLocaleString()} errored · {usd(job.judge_spend ?? 0)} judge spend
               {active && remaining ? ` · ${remaining}` : ""}
             

@@ -386,14 +393,13 @@ const StartForm: React.FC = () => {
   const percentageValid = parsedPct >= 0.1 && parsedPct <= 100;
   const parsedMaxTurns = Number.parseInt(maxTurns, 10);
   const maxTurnsValid = parsedMaxTurns >= 1 && parsedMaxTurns <= 2000;
-  const filled =
-    [apiKeyId, routerName, judgeModel].every((field) => field !== "") &&
-    (direction === "forward" || baselineModel !== "");
+  const baselinePicked = direction === "forward" || baselineModel !== "";
+  const filled = [apiKeyId, routerName, judgeModel].every((field) => field !== "") && baselinePicked;
   const boundsValid = percentageValid && maxTurnsValid;
   const valid = Boolean(accessToken) && filled && boundsValid;
   const handleStart = () => {
     const startBody = {
-      api_key_id: apiKeyId,
+      api_key_ids: [apiKeyId],
       router_name: routerName,
       direction,
       ...(direction === "reverse" ? { baseline_model: baselineModel } : {}),
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts
index 7645fcc3346..eef98320e67 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useShadowEval.ts
@@ -7,6 +7,7 @@ import { $api, fetchClient } from "@/lib/http/api";
 import type { components } from "@/lib/http/schema";
 
 export type ShadowEvalJob = components["schemas"]["ShadowEvalJobResponse"];
+export type ShadowEvalJobKey = components["schemas"]["ShadowEvalJobKeyResponse"];
 export type ShadowEvalSlice = components["schemas"]["ShadowEvalSlice"];
 export type StartShadowEvalRequest = components["schemas"]["StartShadowEvalRequest"];
 
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx
index 029f8aa424c..85350a539c2 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx
@@ -197,11 +197,15 @@ const CostTrackingSettings: React.FC = ({ userID, use
             />
             
               
-                
-                  Discounts
-                  Test It
+                
+                  
+                    Discounts
+                  
+                  
+                    Test It
+                  
                 
-                
+                
                   

                     

                       
@@ -237,7 +241,7 @@ const CostTrackingSettings: React.FC = ({ userID, use
                     )}
                   

                 
-                
+                
                   

                     
                   

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/multi_export_utils.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/multi_export_utils.test.ts
index 3c319e5ba18..25b4da869bb 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/multi_export_utils.test.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/pricing_calculator/multi_export_utils.test.ts
@@ -1,3 +1,5 @@
+// @vitest-environment jsdom
+
 import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
 import { exportMultiToPDF, exportMultiToCSV } from "./multi_export_utils";
 import type { MultiModelResult } from "./types";
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx
index 609af657f20..667c52087f1 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx
@@ -5,7 +5,8 @@ import {
   RollbackOutlined,
   SaveOutlined,
 } from "@ant-design/icons";
-import { Button, Input, Select, Switch } from "antd";
+import { Input, Select, Switch } from "antd";
+import { Button } from "@/components/ui/button";
 import React, { useState } from "react";
 
 interface GuardrailConfigProps {
@@ -56,13 +57,17 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar
               options={versions.map((v) => ({ value: v.id, label: v.label }))}
               style={{ width: 140 }}
             />
-            
           

           

-            
-            
+            
           

@@ -194,12 +199,8 @@ export function GuardrailConfig({ guardrailName, guardrailType, provider }: Guar
         

 
         

-          
 
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
index e80ddac932f..f89e8a3248c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
@@ -5,10 +5,9 @@ import {
   updateGuardrailCall,
 } from "@/components/networking";
 import { copyToClipboard as utilCopyToClipboard } from "@/utils/dataUtils";
-import { CodeOutlined, EyeInvisibleOutlined, InfoCircleOutlined, StopOutlined } from "@ant-design/icons";
+import { EyeInvisibleOutlined, InfoCircleOutlined, StopOutlined } from "@ant-design/icons";
 
-import { Button as AntdButton } from "antd";
-import { ArrowLeft, CheckIcon, CopyIcon } from "lucide-react";
+import { ArrowLeft, CheckIcon, Code, CopyIcon } from "lucide-react";
 import { Badge } from "@/components/ui/badge";
 import { Card } from "@/components/ui/card";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
@@ -511,24 +510,26 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose,
   return (
     

       

-        } onClick={onClose} className="mb-4">
+        
         
{guardrailData.guardrail_name || "Unnamed Guardrail"}

         

           
{guardrailData.guardrail_id}

 
-           : }
+          
         

       

 
@@ -628,13 +629,14 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose,
               
                 

                   

-                    
+                    
                     
Custom Code

                   

                   {isAdmin && !isConfigGuardrail && (
-                    } onClick={() => setCustomCodeModalVisible(true)}>
+                    
                   )}
                 

                 

@@ -671,11 +673,14 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose,
                   {!isEditing &&
                     !isConfigGuardrail &&
                     (guardrailData.litellm_params?.guardrail === "custom_code" ? (
-                      } onClick={() => setCustomCodeModalVisible(true)}>
+                      
                     ) : (
-                       setIsEditing(true)}>Edit Settings
+                      
                     ))}
                 

 
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx
index 539a06910c1..23a55d2dc1a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/EnvVarsSection.tsx
@@ -1,5 +1,6 @@
 import React from "react";
-import { Input, Select, Button, Tooltip, Typography } from "antd";
+import { Input, Select, Tooltip, Typography } from "antd";
+import { Button } from "@/components/ui/button";
 import { InfoCircleOutlined, MinusCircleOutlined, PlusOutlined } from "@ant-design/icons";
 import { useFieldArray, useFormContext, useWatch } from "react-hook-form";
 
@@ -106,7 +107,8 @@ const EnvVarsSection: React.FC = () => {
             

           

         ))}
-        
       

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx
index 3711140562f..80a82dc22b8 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/MCPPermissionManagement.tsx
@@ -1,5 +1,8 @@
 import React, { useEffect } from "react";
-import { Alert, Select, Tooltip, Collapse, Input, Space, Button, Switch } from "antd";
+import { Select, Tooltip, Collapse, Input, Space, Switch } from "antd";
+import { TriangleAlert } from "lucide-react";
+import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert";
+import { Button } from "@/components/ui/button";
 import { InfoCircleOutlined, MinusCircleOutlined, PlusOutlined } from "@ant-design/icons";
 import { useFieldArray, useFormContext, useWatch } from "react-hook-form";
 import { MCPServer, AUTH_TYPE } from "@/components/mcp_tools/types";
@@ -77,7 +80,8 @@ const StaticHeadersFieldArray: React.FC = () => {
           />
         
       ))}
-
@@ -269,13 +273,15 @@ const MCPPermissionManagement: React.FC = ({ )} {showInternalDelegatePkceWarning && ( - + + + Internal server with upstream OAuth delegation + + This MCP server is configured as internal-only but delegates auth to upstream. Anonymous users will be + able to reach the upstream OAuth2 /authorize flow without a LiteLLM session. Ensure your upstream + provider and network enforce access controls. + + )} )}
) : isError ? ( - + + + Failed to load env vars + ) : required.length === 0 ? ( - + + + No per-user fields configured for this server. + ) : ( <> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts index e2e7814f0d7..94544ec6e2d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/createOAuthUiState.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { beforeEach, describe, expect, it, vi } from "vitest"; import { setSecureItem } from "@/utils/secureStorage"; import { CreateUiSnapshot, readCreateUiSnapshot, writeCreateUiSnapshot } from "./createOAuthUiState"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_connect.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_connect.tsx index 066b26a9832..8aeb94404f2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_connect.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_connect.tsx @@ -1,9 +1,22 @@ /* eslint-disable react/no-unescaped-entities */ import React, { useState } from "react"; -import { Card, Typography, Space, Alert, Button, Switch } from "antd"; +import { Card, Typography, Space, Switch } from "antd"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; +import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { CopyIcon, Code, Terminal, Globe, CheckIcon, ExternalLinkIcon, KeyIcon, ServerIcon, Zap } from "lucide-react"; +import { + CopyIcon, + Code, + Terminal, + Globe, + CheckIcon, + ExternalLinkIcon, + Info, + KeyIcon, + ServerIcon, + Zap, +} from "lucide-react"; import { getProxyBaseUrl } from "@/components/networking"; import { copyToClipboard as utilCopyToClipboard } from "@/utils/dataUtils"; @@ -68,25 +81,21 @@ const FeatureCard: React.FC = ({
{useServerHeader && ( - -

- Option 1: Get a specific server: "{serverName.replace(/\s+/g, "_")}" -

-

- Option 2: Get a group of MCPs: "dev-group" -

-

- You can also mix both: "Server1,dev-group" -

- - } - /> + + + Two Options + +

+ Option 1: Get a specific server: "{serverName.replace(/\s+/g, "_")}" +

+

+ Option 2: Get a group of MCPs: "dev-group" +

+

+ You can also mix both: "Server1,dev-group" +

+ + )} )} @@ -145,16 +154,17 @@ const MCPConnect: React.FC = ({ currentServerAccessGroups = [] )} 
{code}
 @@ -441,11 +451,18 @@ const MCPConnect: React.FC = ({ currentServerAccessGroups = [] />
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx index 3892b9d37b7..94b66f1bdfa 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_edit.tsx @@ -1,5 +1,7 @@ import React, { useState, useEffect } from "react"; -import { Select, Button as AntdButton, Tooltip, Input, InputNumber, Alert } from "antd"; +import { Select, Tooltip, Input, InputNumber } from "antd"; +import { TriangleAlert } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { FormProvider, useForm } from "react-hook-form"; import { InfoCircleOutlined } from "@ant-design/icons"; import { Button } from "@/components/ui/button"; @@ -1041,13 +1043,15 @@ const MCPServerEdit: React.FC = ({ {!isStdioTransport && isOAuthAuthType && ( <> {!oauthFlowTypeValue && !isDelegateAuth && ( - + + + This server has no OAuth flow set + + Choose Machine-to-Machine (M2M) or Interactive (PKCE) so LiteLLM authenticates it the way you + intend, then save. Until it is set, LiteLLM falls back to interactive per-user auth and treats + a machine-to-machine credential shape conservatively. + + )} = ({
- Cancel +
@@ -1284,7 +1290,9 @@ const MCPServerEdit: React.FC = ({
- Cancel +
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx index 736327409d6..9c5396d1e9a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_server_view.tsx @@ -130,22 +130,22 @@ export const MCPServerView: React.FC = ({ setSelectedTabIndex(Number(v))}> - - + + Overview - + MCP Tools {isProxyAdmin && ( - + Settings )} {/* Overview Panel */} - +

Transport

@@ -192,7 +192,7 @@ export const MCPServerView: React.FC = ({ {/* Tool Panel */} - + = ({ {/* Settings Panel */} - +

MCP Server Settings

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx index 970fb56799d..ed0c77064b9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/mcp-servers/_components/mcp_servers.tsx @@ -543,7 +543,7 @@ const MCPServers: React.FC = ({ accessToken, userRole, userID }) )} - + {selectedServerId ? ( = ({ accessToken, userRole, userID })
)} - + - + {isAdminRole(userRole) && ( - + )} {isAdminRole(userRole) && ( - + )} {isAdminRole(userRole) && ( - + )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx index 7fe67fb9a41..aca1db0fbae 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx @@ -56,6 +56,8 @@ interface GlobalActivityData { daily_data: { date: string; api_requests: number; total_tokens: number }[]; } +const EMPTY_GLOBAL_ACTIVITY: GlobalActivityData = { sum_api_requests: 0, sum_total_tokens: 0, daily_data: [] }; + type UsageDateRange = { from?: Date; to?: Date }; type TeamSpendTotal = { name: string; value: number }; @@ -105,7 +107,7 @@ const UsagePage: React.FC = ({ accessToken, token, userRole, use const [uniqueTeamIds, setUniqueTeamIds] = useState([]); const [totalSpendPerTeam, setTotalSpendPerTeam] = useState([]); const [spendByProvider, setSpendByProvider] = useState([]); - const [globalActivity, setGlobalActivity] = useState({} as GlobalActivityData); + const [globalActivity, setGlobalActivity] = useState(EMPTY_GLOBAL_ACTIVITY); const [globalActivityPerModel, setGlobalActivityPerModel] = useState([]); const [selectedKeyToken, setSelectedKeyToken] = useState(null); const [selectedTags, setSelectedTags] = useState([ALL_TAGS]); @@ -560,14 +562,14 @@ const UsagePage: React.FC = ({ accessToken, token, userRole, use )} - + Cost Activity - +

@@ -671,7 +673,7 @@ const UsagePage: React.FC = ({ accessToken, token, userRole, use

- +
@@ -751,7 +753,7 @@ const UsagePage: React.FC = ({ accessToken, token, userRole, use - +
@@ -782,7 +784,7 @@ const UsagePage: React.FC = ({ accessToken, token, userRole, use
- +

Customers of your LLM API calls. Tracked when a `user` param is passed in your LLM calls{" "} = ({ accessToken, token, userRole, use - +

- + - + - + - + = ({ {selectedMode === "flow_builder" && ( + > + + You'll be redirected to the full-screen Flow Builder to design your policy logic visually. + + )}
@@ -457,35 +461,33 @@ const AddPolicyForm: React.FC = ({ {resolvedGuardrails.length > 0 && ( - - - These are the final guardrails that will be applied (including inheritance): - -
- {resolvedGuardrails.map((g) => ( - - {g} - - ))} -
+ + + Resolved Guardrails + + + These are the final guardrails that will be applied (including inheritance): + +
+ {resolvedGuardrails.map((g) => ( + + {g} + + ))}
- } - type="info" - showIcon - /> + + )} - + + + Model Scope + + By default, this policy will run on all models. You can optionally restrict it to specific models below. + +
Model Condition Type diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/index.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/index.tsx index 6b8d7c053f2..08ee2213413 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/index.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/index.tsx @@ -409,22 +409,22 @@ const PoliciesPanel: React.FC = ({ accessToken, userRole }) return (
- - + + Templates - + Policies - + Attachments - + Policy Simulator - + = ({ accessToken, userRole }) /> - +
@@ -503,7 +503,7 @@ const PoliciesPanel: React.FC = ({ accessToken, userRole }) /> - + }>

Policy attachments control where your policies apply. Policies don't do anything until you attach @@ -571,7 +571,7 @@ const PoliciesPanel: React.FC = ({ accessToken, userRole }) /> - + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.tsx index 90642d23a46..9c234a7a70c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/policy_test_panel.tsx @@ -1,6 +1,8 @@ import React, { useState, useEffect } from "react"; import { useForm } from "react-hook-form"; -import { Alert, Empty } from "antd"; +import { Empty } from "antd"; +import { CircleAlert } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { resolvePoliciesCall, teamListCall, keyListCall, modelAvailableCall } from "@/components/networking"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { FieldGroup } from "@/components/shared/form/field"; @@ -325,7 +327,11 @@ const PolicyTestPanel: React.FC = ({ accessToken }) => { )} {hasSearched && !result && !isLoading && ( - + + + Error + Failed to resolve policies. Check the proxy logs. + )}

); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx index cf29280ea51..5ce91f15bd4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/prompts/_components/prompt_info.tsx @@ -4,7 +4,7 @@ import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { Button as AntButton, Modal } from "antd"; +import { Modal } from "antd"; import { getPromptInfo, getPromptVersions, @@ -191,17 +191,18 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo

Prompt Details

{basePromptId}

- : } +
@@ -412,10 +413,9 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo

Prompt Template

- : } +
@@ -462,10 +463,9 @@ const PromptInfoView: React.FC = ({ promptId, onClose, accessTo

Raw API Response

- : } +
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx index 2a5b94b2fd7..1e58e81f4e3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/router-settings/_components/general_settings.tsx @@ -261,19 +261,19 @@ const GeneralSettings: React.FC = ({ accessToken, user Prompt Caching General - + - + - + - + - + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/tag-management/_components/tag_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/tag-management/_components/tag_info.tsx index 5c561f38197..f05bddb3f6b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/tag-management/_components/tag_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/tag-management/_components/tag_info.tsx @@ -2,7 +2,6 @@ import React, { useState, useEffect } from "react"; import { SimpleTooltip } from "@/components/ui/tooltip"; -import { Button as AntdButton } from "antd"; import { z } from "zod/v4"; import { fetchUserModels } from "@/components/organisms/create_key_button"; import { getModelDisplayName } from "@/components/key_team_helpers/fetch_available_models_team_key"; @@ -230,17 +229,18 @@ const TagInfoView: React.FC = ({ tagId, onClose, accessToken, {tagDetails.name} - : } +

{tagDetails.description || "No description"}

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx index 5ea15df13f0..9fb4dad67e4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.tsx @@ -18,7 +18,7 @@ import { Member, } from "@/components/networking"; import { SimpleTooltip } from "@/components/ui/tooltip"; -import { Button as AntdButton, Modal } from "antd"; +import { Modal } from "antd"; import { Field, FieldGroup, FieldLabel } from "@/components/shared/form/field"; import { Combobox, @@ -405,17 +405,18 @@ export default function UserInfoView({

{userData.user_email || "User"}

{userData.user_id} - : } +
{userRole && rolesWithWriteAccess.includes(userRole) && ( @@ -585,17 +586,18 @@ export default function UserInfoView({

User ID

{userData.user_id} - : } +
@@ -760,9 +762,9 @@ export default function UserInfoView({
- +
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx index 93aaa90ef7c..271ef31a68e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.tsx @@ -1,9 +1,10 @@ import React, { useState } from "react"; -import { Upload, Alert } from "antd"; +import { Upload } from "antd"; import { toast } from "@/lib/toast"; import { InboxOutlined } from "@ant-design/icons"; import type { UploadProps } from "antd"; -import { CircleHelp } from "lucide-react"; +import { CircleCheck, CircleHelp, X } from "lucide-react"; +import { Alert, AlertAction, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { ragIngestCall } from "@/components/networking"; import { DocumentUpload, RAGIngestResponse } from "@/components/vector_store_management/types"; import DocumentsTable from "./DocumentsTable"; @@ -359,22 +360,23 @@ const CreateVectorStore: React.FC = ({ accessToken, onSu {/* Success Message */} {ingestResults.length > 0 && ( - -

- Vector Store ID: {ingestResults[0]?.vector_store_id} -

-

- Documents Ingested: {ingestResults.length} -

- - } - type="success" - showIcon - closable - /> + + + Vector Store Created Successfully + +

+ Vector Store ID: {ingestResults[0]?.vector_store_id} +

+

+ Documents Ingested: {ingestResults.length} +

+ + + + + )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx index 3e286ee9b92..cdc63e8b7ec 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/S3VectorsConfig.tsx @@ -1,6 +1,6 @@ import React, { useState, useEffect } from "react"; -import { Alert } from "antd"; -import { CircleHelp } from "lucide-react"; +import { CircleHelp, Info } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; import { Field, FieldError, FieldLabel } from "@/components/shared/form/field"; import { @@ -72,32 +72,28 @@ const S3VectorsConfig: React.FC = ({ accessToken, provider return ( - -

AWS S3 Vectors allows you to store and query vector embeddings directly in S3:

-
    -
  • Vector buckets and indexes will be automatically created if they don't exist
    • -
  • Vector dimensions are auto-detected from your selected embedding model
    • -
  • Ensure your AWS credentials have permissions for S3 Vectors operations
    • -
  • - Learn more:{" "} - - AWS S3 Vectors Documentation - -
    • -
- - } - type="info" - showIcon - style={{ marginBottom: "16px" }} - /> + + + AWS S3 Vectors Setup + +

AWS S3 Vectors allows you to store and query vector embeddings directly in S3:

+
    +
  • Vector buckets and indexes will be automatically created if they don't exist
    • +
  • Vector dimensions are auto-detected from your selected embedding model
    • +
  • Ensure your AWS credentials have permissions for S3 Vectors operations
    • +
  • + Learn more:{" "} + + AWS S3 Vectors Documentation + +
    • +
+ + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx index 05c829d79df..ff6c10bf428 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx @@ -1,6 +1,7 @@ import React, { useState, useEffect } from "react"; -import { Modal, Alert } from "antd"; -import { CircleHelp, Eye, EyeOff } from "lucide-react"; +import { Modal } from "antd"; +import { CircleHelp, Eye, EyeOff, Info } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { useWatch } from "react-hook-form"; import { z } from "zod/v4"; import { CredentialItem, vectorStoreCreateCall } from "@/components/networking"; @@ -310,142 +311,129 @@ const VectorStoreForm: React.FC = ({ {selectedProvider === "pg_vector" && ( - -

LiteLLM provides a server to connect to PG Vector. To use this provider:

-
    -
  1. - Deploy the litellm-pgvector server from:{" "} - - https://github.com/BerriAI/litellm-pgvector - -
    2. -
  2. Configure your PostgreSQL database with pgvector extension
    3. -
  3. Start the server and note the API base URL and API key
    4. -
  4. Enter those details in the fields below
    5. -
- - } - type="info" - showIcon - /> + + + PG Vector Setup Required + +

LiteLLM provides a server to connect to PG Vector. To use this provider:

+
    +
  1. + Deploy the litellm-pgvector server from:{" "} + + https://github.com/BerriAI/litellm-pgvector + +
    2. +
  2. Configure your PostgreSQL database with pgvector extension
    3. +
  3. Start the server and note the API base URL and API key
    4. +
  4. Enter those details in the fields below
    5. +
+ + )} {selectedProvider === "valkey" && ( - -

- LiteLLM searches documents you have already stored in Valkey. It does not create the index or - upload documents for you. Before creating this vector store, make sure: -

-
    -
  1. - Your Valkey server has vector search enabled (the valkey-search module, included in the - valkey-bundle image and in AWS ElastiCache / MemoryDB for Valkey) -
    2. -
  2. - You have already created a search index and loaded your documents and their embeddings into it. - Enter that index name as the Vector Store ID -
    3. -
  3. - You know which embedding model created those stored embeddings. That model must be added to this - proxy under Models so you can pick it below. Using a different model returns wrong results -
    4. -
  4. - You know the field names your documents use for their text and their embedding. If they are not - "text" and "embedding", set them below -
    5. -
-

- When a query comes in, LiteLLM converts it to an embedding with the model below and returns the - closest matching documents from your index. -

- - } - type="info" - showIcon - /> + + + Valkey Setup Required + +

+ LiteLLM searches documents you have already stored in Valkey. It does not create the index or upload + documents for you. Before creating this vector store, make sure: +

+
    +
  1. + Your Valkey server has vector search enabled (the valkey-search module, included in the + valkey-bundle image and in AWS ElastiCache / MemoryDB for Valkey) +
    2. +
  2. + You have already created a search index and loaded your documents and their embeddings into it. + Enter that index name as the Vector Store ID +
    3. +
  3. + You know which embedding model created those stored embeddings. That model must be added to this + proxy under Models so you can pick it below. Using a different model returns wrong results +
    4. +
  4. + You know the field names your documents use for their text and their embedding. If they are not + "text" and "embedding", set them below +
    5. +
+

+ When a query comes in, LiteLLM converts it to an embedding with the model below and returns the + closest matching documents from your index. +

+ + )} {selectedProvider === "vertex_rag_engine" && ( - -

To use Vertex AI RAG Engine:

-

- Note: Google Cloud has renamed this to "RAG Engine" in its console — the steps below - still apply. -

-
    -
  1. - Set up your Vertex AI RAG Engine corpus following the guide:{" "} - - Vertex AI RAG Engine Overview - -
    2. -
  2. Create a corpus in your Google Cloud project
    3. -
  3. - Note the corpus ID from the Vertex AI console (now labeled "RAG Engine" in Google - Cloud) -
    4. -
  4. Enter the corpus ID in the Vector Store ID field below
    5. -
- - } - type="info" - showIcon - /> + + + Vertex AI RAG Engine Setup + +

To use Vertex AI RAG Engine:

+

+ Note: Google Cloud has renamed this to "RAG Engine" in its console — the steps below still + apply. +

+
    +
  1. + Set up your Vertex AI RAG Engine corpus following the guide:{" "} + + Vertex AI RAG Engine Overview + +
    2. +
  2. Create a corpus in your Google Cloud project
    3. +
  3. + Note the corpus ID from the Vertex AI console (now labeled "RAG Engine" in Google Cloud) +
    4. +
  4. Enter the corpus ID in the Vector Store ID field below
    5. +
+ + )} {selectedProvider === "vertex_ai/search_api" && ( - -

To use Vertex AI Search (Discovery Engine):

-

- Note: Google Cloud has renamed this to "Agent Search" in its console — the steps below - still apply. -

-
    -
  1. - Enable the Discovery Engine API on your Google Cloud project and create a data store following - the guide:{" "} - - Create a Vertex AI Search data store - -
    2. -
  2. Pick a supported location: global, us, or eu
    3. -
  3. - For most data store types (Cloud Storage, BigQuery, Media): copy the data store ID and enter it - in the Vector Store ID field below. -
    4. -
  4. - For website, healthcare, and connector-based sources (Drive, Gmail, Slack, Jira, etc.): create a - search app on top of the data store, then copy the Engine ID and enter it in - the Engine ID field. The Vector Store ID is still required as the LiteLLM-side name for this - record, but it isn't used in the GCP URL when Engine ID is set. -
    5. -
- - } - type="info" - showIcon - /> + + + Vertex AI Search Setup + +

To use Vertex AI Search (Discovery Engine):

+

+ Note: Google Cloud has renamed this to "Agent Search" in its console — the steps below + still apply. +

+
    +
  1. + Enable the Discovery Engine API on your Google Cloud project and create a data store following the + guide:{" "} + + Create a Vertex AI Search data store + +
    2. +
  2. Pick a supported location: global, us, or eu
    3. +
  3. + For most data store types (Cloud Storage, BigQuery, Media): copy the data store ID and enter it in + the Vector Store ID field below. +
    4. +
  4. + For website, healthcare, and connector-based sources (Drive, Gmail, Slack, Jira, etc.): create a + search app on top of the data store, then copy the Engine ID and enter it in the + Engine ID field. The Vector Store ID is still required as the LiteLLM-side name for this record, + but it isn't used in the GCP URL when Engine ID is set. +
    5. +
+ + )} ; +function SsoEnabledNotice() { + const [isDismissed, setIsDismissed] = useState(false); + if (isDismissed) return null; + + return ( + + + + Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow upon loading + this page. To re-enable auto-redirect-to-SSO, set{" "} + AUTO_REDIRECT_UI_LOGIN_TO_SSO=true in your + environment configuration. + + + + + + ); +} + function LoginPageContent() { const [isLoading, setIsLoading] = useState(true); const { data: uiConfig, isLoading: isConfigLoading } = useUIConfig(); @@ -170,22 +192,19 @@ function LoginPageContent() {

🚅 LiteLLM

- -

- The Admin UI has been disabled by the administrator. To re-enable it, please update the following - environment variable: -

-

- DISABLE_ADMIN_UI=False -

- - } - type="warning" - showIcon - /> + + + Admin UI Disabled + +

+ The Admin UI has been disabled by the administrator. To re-enable it, please update the following + environment variable: +

+

+ DISABLE_ADMIN_UI=False +

+ + @@ -209,31 +228,32 @@ function LoginPageContent() { {!uiConfig?.hide_default_credentials_hint && ( - -

- By default, Username is admin{" "} - and Password is your set LiteLLM Proxy - MASTER_KEY. -

-

- Need to set UI credentials or SSO?{" "} - - Check the documentation - - . -

- - } - type="info" - icon={} - showIcon - /> + + + Default Credentials + +

+ By default, Username is admin and + Password is your set LiteLLM Proxy + MASTER_KEY. +

+

+ Need to set UI credentials or SSO?{" "} + + Check the documentation + + . +

+ + )} - {error && } + {error && ( + + + {error} + + )}
@@ -326,22 +346,7 @@ function LoginPageContent() { - {uiConfig?.sso_configured && ( - - Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow - upon loading this page. To re-enable auto-redirect-to-SSO, set{" "} - AUTO_REDIRECT_UI_LOGIN_TO_SSO=true{" "} - in your environment configuration. - - } - /> - )} + {uiConfig?.sso_configured && } diff --git a/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx b/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx index 1f2441c4870..0a03cc6a591 100644 --- a/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx +++ b/ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.tsx @@ -1,6 +1,7 @@ -import { Alert } from "antd"; +import { CircleAlert, Info } from "lucide-react"; import React from "react"; import { z } from "zod/v4"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { PasswordInput } from "@/components/shared/PasswordInput"; import { Field, FieldLabel, FieldGroup } from "@/components/shared/form/field"; import { FormField } from "@/components/shared/form/FormField"; @@ -46,11 +47,10 @@ export function OnboardingFormBody({ variant, userEmail, isPending, claimError,

{variant === "signup" && ( - + + SSO +
SSO is under the Enterprise Tier.
- } - showIcon - /> + + )}
@@ -84,7 +83,12 @@ export function OnboardingFormBody({ variant, userEmail, isPending, claimError, - {claimError && } + {claimError && ( + + + {claimError} + + )}
diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.tsx index 89b0f369720..a57ea763b6c 100644 --- a/ui/litellm-dashboard/src/components/CreateUserButton.tsx +++ b/ui/litellm-dashboard/src/components/CreateUserButton.tsx @@ -10,8 +10,9 @@ import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Textarea } from "@/components/ui/textarea"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; -import { Alert, Modal, Typography } from "antd"; -import { ChevronRight, CircleHelp, UserPlus } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; +import { Modal, Typography } from "antd"; +import { ChevronRight, CircleHelp, Info, UserPlus } from "lucide-react"; import React, { useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import TeamDropdown from "./common_components/team_dropdown"; @@ -134,20 +135,16 @@ const labelWithHint = (label: string, hint: string): React.ReactNode => ( ); const EmailInvitationsNotice: React.FC = () => ( - - New users receive an email invite only when an email integration (SMTP, Resend, or SendGrid) is configured.{" "} - - Learn how to set up email notifications - - - } - type="info" - showIcon - className="mb-4" - /> + + + Email invitations + + New users receive an email invite only when an email integration (SMTP, Resend, or SendGrid) is configured.{" "} + + Learn how to set up email notifications + + + ); export const CreateUserButton: React.FC = ({ diff --git a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts index d80294aa2e6..7ed014d43b2 100644 --- a/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts +++ b/ui/litellm-dashboard/src/components/EntityUsageExport/utils.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; import Papa from "papaparse"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx index d8305171447..ba0b3a5037a 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/MCPSemanticFilterSettings/MCPSemanticFilterSettings.tsx @@ -3,9 +3,9 @@ import { useMCPSemanticFilterSettings } from "@/app/(dashboard)/hooks/mcpSemanticFilterSettings/useMCPSemanticFilterSettings"; import { useUpdateMCPSemanticFilterSettings } from "@/app/(dashboard)/hooks/mcpSemanticFilterSettings/useUpdateMCPSemanticFilterSettings"; import { toast } from "@/lib/toast"; -import { Alert, Card, Col, Row, Skeleton } from "antd"; -import { CheckCircleOutlined } from "@ant-design/icons"; -import { CircleHelp, Save } from "lucide-react"; +import { Card, Col, Row, Skeleton } from "antd"; +import { CircleCheck, CircleHelp, Info, Save, X } from "lucide-react"; +import { Alert, AlertAction, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models"; @@ -177,40 +177,39 @@ export default function MCPSemanticFilterSettings({ accessToken }: MCPSemanticFi {isLoading ? ( ) : isError ? ( - + + Could not load MCP Semantic Filter settings + {error instanceof Error && {error.message}} + ) : ( <> - + + + Semantic Tool Filtering + + Filter MCP tools semantically based on query relevance. This reduces context window size and improves tool + selection accuracy. Click 'Save Settings' to apply changes across all pods (takes effect within + 10 seconds). + + {saveSuccess && ( - } - showIcon - closable - style={{ marginBottom: 16 }} - /> + + + Settings saved successfully + + + + )} {updateError && ( - + + Could not update settings + {updateError instanceof Error && {updateError.message}} + )} diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/PluginSettings/PluginSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/PluginSettings/PluginSettings.tsx index c93cf812738..fc085ebbd9e 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/PluginSettings/PluginSettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/PluginSettings/PluginSettings.tsx @@ -1,7 +1,8 @@ "use client"; import { useState, useEffect } from "react"; -import { Button, Card, Modal, Space, Table, Typography } from "antd"; +import { Card, Modal, Space, Table, Typography } from "antd"; +import { Button } from "@/components/ui/button"; import { DeleteOutlined, EditOutlined, PlusOutlined } from "@ant-design/icons"; import { Eye, EyeOff } from "lucide-react"; import { getConfigFieldSetting, updateConfigFieldSetting } from "@/components/networking"; @@ -113,8 +114,12 @@ export default function PluginSettings() { key: "actions", render: (_: unknown, __: Plugin, idx: number) => ( - + ), }, @@ -131,7 +136,8 @@ export default function PluginSettings() { Each plugin must expose GET /api/plugin-manifest returning nav items and capabilities. - diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx index 4d339737ab0..c0c73fbb762 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.tsx @@ -5,7 +5,10 @@ import { all_admin_roles, isUserTeamAdminForAnyTeam } from "@/utils/roles"; import { modelCreationScope } from "@/utils/modelPermissions"; import { Switch } from "@/components/ui/switch"; import { Field, FieldLabel } from "@/components/shared/form/field"; -import { Select as AntdSelect, Button, Card, Col, Modal, Row, Tooltip, Typography, Alert } from "antd"; +import { Select as AntdSelect, Card, Col, Modal, Row, Tooltip, Typography } from "antd"; +import { Info } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; +import { Button } from "@/components/ui/button"; import type { UploadProps } from "antd/es/upload"; import React, { useEffect, useMemo, useState } from "react"; import { FormProvider, useWatch, type UseFormReturn } from "react-hook-form"; @@ -167,13 +170,13 @@ const AddModelForm: React.FC = ({ )} {!teamAdminSelectedTeam && ( - + + + Team Selection Required + + As a team admin, you need to select your team first before adding models. + + )} )} @@ -418,10 +421,16 @@ const AddModelForm: React.FC = ({ Need Help?
- -
@@ -443,6 +452,7 @@ const AddModelForm: React.FC = ({ footer={[
- @@ -131,12 +133,14 @@ const KeywordTierRules: React.FC = ({ rules, onChange, ti /> ))} diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index 955167c4351..e5f1f108b4f 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -1,7 +1,8 @@ import { useProviderFields } from "@/app/(dashboard)/hooks/providers/useProviderFields"; import { UploadOutlined } from "@ant-design/icons"; import { Input } from "@/components/ui/input"; -import { Button as Button2, Col, Input as AntdInput, Row, Select, Typography, Upload, UploadProps } from "antd"; +import { Col, Input as AntdInput, Row, Select, Typography, Upload, UploadProps } from "antd"; +import { Button } from "@/components/ui/button"; import React from "react"; import { useFormContext } from "react-hook-form"; import { antdRequired } from "../common_components/antdFormRules"; @@ -255,7 +256,10 @@ const ProviderSpecificFields: React.FC = ({ selecte } }} > - }>Click to Upload + ); } diff --git a/ui/litellm-dashboard/src/components/add_pass_through.tsx b/ui/litellm-dashboard/src/components/add_pass_through.tsx index 9f95e7f27ac..09ddac2cf42 100644 --- a/ui/litellm-dashboard/src/components/add_pass_through.tsx +++ b/ui/litellm-dashboard/src/components/add_pass_through.tsx @@ -1,8 +1,9 @@ "use client"; import React, { useState } from "react"; -import { Modal, Alert } from "antd"; -import { CircleHelp, Plug } from "lucide-react"; +import { Modal } from "antd"; +import { CircleHelp, Info, Plug } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { useWatch } from "react-hook-form"; import { z } from "zod/v4"; @@ -156,13 +157,14 @@ const AddPassThroughEndpoint: React.FC = ({ }} >
- + + + What is a Pass-Through Endpoint? + + Route requests from your LiteLLM proxy to any external API. Perfect for custom models, image generation + APIs, or any service you want to proxy through LiteLLM. + + diff --git a/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx b/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx index f51629b7ee1..a544a8be32b 100644 --- a/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx +++ b/ui/litellm-dashboard/src/components/common_components/PassThroughGuardrailsSection.tsx @@ -1,6 +1,6 @@ import React from "react"; -import { Alert } from "antd"; -import { CircleHelp } from "lucide-react"; +import { CircleHelp, Info } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import GuardrailSelector from "../guardrails/GuardrailSelector"; import { TagsInput } from "@/app/(dashboard)/guardrails/_components/content_filter/TagsInput"; @@ -67,21 +67,20 @@ const PassThroughGuardrailsSection: React.FC endpoints.

- - Field-Level Targeting{" "} - - (Learn More) - - - } - description={ + + + + Field-Level Targeting{" "} + + (Learn More) + + +
Optionally specify which fields to check. If left empty, the entire request/response is sent to the @@ -100,11 +99,8 @@ const PassThroughGuardrailsSection: React.FC
- } - type="info" - showIcon - className="mb-4" - /> + + diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx index 6b71d5ab1d2..dfe6d762ac6 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.test.tsx @@ -74,7 +74,7 @@ describe("UserSearchModal", () => { expect(notice).toHaveTextContent(/users that already exist/i); expect(notice).toHaveTextContent(/ask a proxy admin to create their account first/i); // info, not warning: a warning here would read as an error state on an empty form - expect(notice.className).toMatch(/ant-alert-info/); + expect(notice.className).toMatch(/text-info/); }); }); diff --git a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx index 41cbd58ba53..547e19da738 100644 --- a/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx +++ b/ui/litellm-dashboard/src/components/common_components/user_search_modal.tsx @@ -1,5 +1,7 @@ import { useRef, useState } from "react"; -import { Modal, Alert } from "antd"; +import { Modal } from "antd"; +import { Info } from "lucide-react"; +import { Alert, AlertTitle } from "@/components/shared/Alert"; import { UserAddOutlined } from "@ant-design/icons"; import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer"; import { useForm } from "react-hook-form"; @@ -209,13 +211,13 @@ const UserSearchModal: React.FC = ({ - + + + + Search selects from users that already exist. To add someone new, ask a proxy admin to create their + account first. + + diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx index 062aaaa1773..1d327b87346 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/BudgetFallbacksEditor.tsx @@ -1,5 +1,6 @@ import { SimpleTooltip } from "@/components/ui/tooltip"; -import { Button, Select } from "antd"; +import { Select } from "antd"; +import { Button } from "@/components/ui/button"; import { ArrowDown, Plus, X } from "lucide-react"; import React, { useState } from "react"; @@ -62,7 +63,8 @@ export function BudgetFallbacksEditor({ value, onChange, availableModels }: Budg
When a model exceeds its per-model budget, requests automatically reroute to fallback models
- @@ -143,7 +145,8 @@ export function BudgetFallbacksEditor({ value, onChange, availableModels }: Budg ); })} - diff --git a/ui/litellm-dashboard/src/components/key_team_helpers/BudgetWindowsEditor.tsx b/ui/litellm-dashboard/src/components/key_team_helpers/BudgetWindowsEditor.tsx index 0c90674a479..c9a422dcb85 100644 --- a/ui/litellm-dashboard/src/components/key_team_helpers/BudgetWindowsEditor.tsx +++ b/ui/litellm-dashboard/src/components/key_team_helpers/BudgetWindowsEditor.tsx @@ -1,4 +1,5 @@ -import { Button, InputNumber, Select } from "antd"; +import { InputNumber, Select } from "antd"; +import { Button } from "@/components/ui/button"; import React from "react"; export interface BudgetWindowEntry { @@ -55,7 +56,12 @@ export function BudgetWindowsEditor({ value, onChange }: BudgetWindowsEditorProp style={{ width: 160 }} prefix="$" /> - @@ -64,7 +70,8 @@ export function BudgetWindowsEditor({ value, onChange }: BudgetWindowsEditorProp ); })} - + diff --git a/ui/litellm-dashboard/src/components/model_dashboard/ModelSettingsModal/ModelSettingsModal.test.tsx b/ui/litellm-dashboard/src/components/model_dashboard/ModelSettingsModal/ModelSettingsModal.test.tsx index fb6624ed92d..ccf5bb062b2 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/ModelSettingsModal/ModelSettingsModal.test.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/ModelSettingsModal/ModelSettingsModal.test.tsx @@ -2,7 +2,7 @@ import { useProxyConfig } from "@/app/(dashboard)/hooks/proxyConfig/useProxyConf import { useStoreModelInDB } from "@/app/(dashboard)/hooks/storeModelInDB/useStoreModelInDB"; import { toast } from "@/lib/toast"; import { parseErrorMessage } from "@/components/shared/errorUtils"; -import { screen, waitFor, within } from "@testing-library/react"; +import { screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders } from "../../../../tests/test-utils"; @@ -225,7 +225,8 @@ describe("ModelSettingsModal", () => { const saveButton = screen.getByRole("button", { name: /Saving/i }); expect(saveButton).toBeInTheDocument(); - expect(within(saveButton).getByRole("img", { name: "loading" })).toBeInTheDocument(); + expect(saveButton).toHaveAttribute("aria-busy", "true"); + expect(saveButton).toBeDisabled(); }); it("should not render modal when isVisible is false", () => { diff --git a/ui/litellm-dashboard/src/components/model_dashboard/ModelSettingsModal/ModelSettingsModal.tsx b/ui/litellm-dashboard/src/components/model_dashboard/ModelSettingsModal/ModelSettingsModal.tsx index 01696e2d820..400b1c2859d 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/ModelSettingsModal/ModelSettingsModal.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/ModelSettingsModal/ModelSettingsModal.tsx @@ -8,7 +8,8 @@ import { FieldGroup } from "@/components/shared/form/field"; import { FormField } from "@/components/shared/form/FormField"; import { Switch } from "@/components/ui/switch"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; -import { Button, Modal, Skeleton, Space, Typography } from "antd"; +import { Modal, Skeleton, Space, Typography } from "antd"; +import { Button } from "@/components/ui/button"; import { CircleHelp } from "lucide-react"; import React, { useEffect, useMemo } from "react"; import { useForm } from "react-hook-form"; @@ -85,13 +86,12 @@ const ModelSettingsModal: React.FC = ({ isVisible, onCa open={isVisible} footer={ -
{(!isAnyAutoRouter || isComplexityRouterModel) && ( - } + )} {!isAnyAutoRouter && ( <> - } + - } + )} - } +
@@ -865,9 +869,9 @@ export default function ModelInfoView({ open={isAutoRouterTestModalOpen} onCancel={() => setIsAutoRouterTestModalOpen(false)} footer={[ - setIsAutoRouterTestModalOpen(false)}> + , ]} width={700} > diff --git a/ui/litellm-dashboard/src/components/onboarding_link.tsx b/ui/litellm-dashboard/src/components/onboarding_link.tsx index 82c99119cf4..583a0d25b29 100644 --- a/ui/litellm-dashboard/src/components/onboarding_link.tsx +++ b/ui/litellm-dashboard/src/components/onboarding_link.tsx @@ -1,5 +1,6 @@ import React from "react"; -import { Button, Modal, Typography } from "antd"; +import { Modal, Typography } from "antd"; +import { Button } from "@/components/ui/button"; import { CopyToClipboard } from "react-copy-to-clipboard"; import { toast } from "@/lib/toast"; @@ -98,9 +99,7 @@ export default function OnboardingModal({
toast.success("Copied!")}> - +
diff --git a/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx b/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx index 2edd502eee3..88554ddba89 100644 --- a/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx +++ b/ui/litellm-dashboard/src/components/organisms/RegenerateKeyModal.tsx @@ -1,7 +1,9 @@ import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { CheckOutlined, CopyOutlined, SyncOutlined } from "@ant-design/icons"; -import { Alert, Button, Modal, Space } from "antd"; -import { CircleHelp } from "lucide-react"; +import { Alert, AlertTitle } from "@/components/shared/Alert"; +import { Modal, Space } from "antd"; +import { Button } from "@/components/ui/button"; +import { CircleHelp, TriangleAlert } from "lucide-react"; import React, { useEffect, useMemo, useState } from "react"; import { useWatch } from "react-hook-form"; import { CopyToClipboard } from "react-copy-to-clipboard"; @@ -157,9 +159,12 @@ export function RegenerateKeyModal({ selectedToken, visible, onClose, onKeyUpdat regeneratedKey ? [ - + - @@ -167,8 +172,11 @@ export function RegenerateKeyModal({ selectedToken, visible, onClose, onKeyUpdat ] : [ - - + , @@ -177,7 +185,10 @@ export function RegenerateKeyModal({ selectedToken, visible, onClose, onKeyUpdat > {regeneratedKey ? (
- + + + Save it now, you will not see it again +
Key Alias diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 62ad62e0f83..b98cc4205bf 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -13,7 +13,7 @@ import { Button } from "@/components/ui/button"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { Input } from "@/components/ui/input"; import { Field, FieldLabel } from "@/components/shared/form/field"; -import { Button as Button2, Input as AntdInput, Modal, Radio, Select, Switch, Tag, Tooltip, Typography } from "antd"; +import { Input as AntdInput, Modal, Radio, Select, Switch, Tag, Tooltip, Typography } from "antd"; import { ChevronDown } from "lucide-react"; import { useDebouncedCallback } from "@tanstack/react-pacer/debouncer"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; @@ -715,9 +715,9 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp style={{ width: "100%" }} notFoundContent={userSearchLoading ? "Searching..." : "No users found"} /> - setIsCreateUserModalVisible(true)} style={{ marginLeft: "8px" }}> +
Search by email to find users
@@ -1733,9 +1733,9 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp )}
- +
diff --git a/ui/litellm-dashboard/src/components/per_user_usage.tsx b/ui/litellm-dashboard/src/components/per_user_usage.tsx index b239da83766..af45facb9a8 100644 --- a/ui/litellm-dashboard/src/components/per_user_usage.tsx +++ b/ui/litellm-dashboard/src/components/per_user_usage.tsx @@ -124,11 +124,11 @@ const PerUserUsage: React.FC = ({ accessToken, selectedTags,

Individual developer usage metrics

- - + + User Details - + Usage Distribution diff --git a/ui/litellm-dashboard/src/components/routing_groups/index.tsx b/ui/litellm-dashboard/src/components/routing_groups/index.tsx index 41e71c77e1f..1ecea5b10da 100644 --- a/ui/litellm-dashboard/src/components/routing_groups/index.tsx +++ b/ui/litellm-dashboard/src/components/routing_groups/index.tsx @@ -1,7 +1,8 @@ "use client"; import React, { useMemo, useState } from "react"; -import { Button, Card, Flex, Input, Modal, Space, Typography } from "antd"; +import { Card, Flex, Input, Modal, Space, Typography } from "antd"; +import { Button } from "@/components/ui/button"; import { PlusOutlined, ReloadOutlined, SearchOutlined } from "@ant-design/icons"; import { useRoutingGroups, useSaveRoutingGroups } from "@/app/(dashboard)/hooks/routingGroups/useRoutingGroups"; import { useRouterFields } from "@/app/(dashboard)/hooks/router/useRouterFields"; @@ -112,10 +113,17 @@ const RoutingGroups: React.FC = () => { className="max-w-sm" /> - - diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx index 2b9d8dad90b..a0bf3afdb20 100644 --- a/ui/litellm-dashboard/src/components/settings.tsx +++ b/ui/litellm-dashboard/src/components/settings.tsx @@ -491,7 +491,7 @@ const Settings: React.FC = ({ accessToken, userRole, userID, Alerting Settings Email Alerts - + = ({ accessToken, userRole, userID, }} /> - +
- +

Alerts are only supported for Slack Webhook URLs. Get your webhook urls from{" "} @@ -601,10 +601,10 @@ const Settings: React.FC = ({ accessToken, userRole, userID, - + - + diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index c20f47a155b..1684c3b1225 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -27,7 +27,7 @@ import { Badge } from "@/components/ui/badge"; import { Card } from "@/components/ui/card"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import { Input as UIInput } from "@/components/ui/input"; -import { Button as UIButton } from "@/components/ui/button"; +import { Button } from "@/components/ui/button"; import { Switch } from "@/components/ui/switch"; import { Textarea } from "@/components/ui/textarea"; import { TooltipProvider } from "@/components/ui/tooltip"; @@ -39,7 +39,7 @@ import { MultiSelect } from "@/components/shared/MultiSelect"; import { SearchSelect } from "@/components/shared/SearchSelect"; import { useZodForm } from "@/lib/forms/useZodForm"; import { TagsInput } from "@/app/(dashboard)/guardrails/_components/content_filter/TagsInput"; -import { Button, Tabs, Tooltip } from "antd"; +import { Tabs, Tooltip } from "antd"; import { toast } from "@/lib/toast"; import { CheckIcon, ChevronDown, CircleMinus, CopyIcon, Plus, Save } from "lucide-react"; import React, { useEffect, useMemo, useState } from "react"; @@ -950,23 +950,25 @@ const TeamInfoView: React.FC = ({

-

{info.team_alias}

{info.team_id}

@@ -1149,12 +1151,13 @@ const TeamInfoView: React.FC = ({

Team Settings

{canEditTeam && !isEditing && ( )} @@ -1444,7 +1447,7 @@ const TeamInfoView: React.FC = ({ /> )} - = ({ onClick={() => removeModelLimit(index)} > - +
))} - = ({ > Add Model Limit - + = ({
- setIsEditing(false)} disabled={isTeamSaving} > Cancel - - + +
diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index c57a76a878a..e33ae8323f5 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -599,9 +599,13 @@ export default function KeyInfoView({ - - Overview - Settings + + + Overview + + + Settings +
diff --git a/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx b/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx index b0a19e7ea05..a436ba55c01 100644 --- a/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx +++ b/ui/litellm-dashboard/src/components/update_model_credentials_modal.tsx @@ -1,10 +1,12 @@ -import { Alert, Modal, Typography } from "antd"; +import { Modal, Typography } from "antd"; +import { TriangleAlert } from "lucide-react"; import { useState } from "react"; import { z } from "zod/v4"; import { modelPatchUpdateCall } from "./networking"; import { toast } from "@/lib/toast"; import { FieldGroup } from "@/components/shared/form/field"; import { FormField } from "@/components/shared/form/FormField"; +import { Alert, AlertTitle } from "@/components/shared/Alert"; import { PasswordInput } from "@/components/shared/PasswordInput"; import { Button } from "@/components/ui/button"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; @@ -74,12 +76,14 @@ export default function UpdateModelCredentialsModal({ Update this model's API key. Only the new key is sent; the rest of the deployment configuration is left untouched. - + + + + Only the API key is rotated here. Models that authenticate with an Azure AD token, AWS credentials, or a + Vertex service-account JSON aren't supported yet; update those from the model's LiteLLM Params for + now. + +
diff --git a/ui/litellm-dashboard/src/components/user_agent_activity.tsx b/ui/litellm-dashboard/src/components/user_agent_activity.tsx index 74716fd866e..4a8b1d11204 100644 --- a/ui/litellm-dashboard/src/components/user_agent_activity.tsx +++ b/ui/litellm-dashboard/src/components/user_agent_activity.tsx @@ -485,11 +485,11 @@ const UserAgentActivity: React.FC = ({ accessToken, user - - + + DAU/WAU/MAU - + Per User Usage (Last 30 Days) @@ -502,14 +502,14 @@ const UserAgentActivity: React.FC = ({ accessToken, user
- - + + DAU - + WAU - + MAU diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 7ec6c98a82c..4c63586d4d6 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -816,7 +816,8 @@ export interface paths { }; /** * List Shadow Eval Jobs - * @description List shadow eval jobs, newest first. Counts and results ride the detail endpoint only. + * @description List shadow eval jobs, newest first, each key with its attempt count so status is + * accurate. Judged counts, spend, and results ride the detail endpoint only. */ get: operations["list_shadow_eval_jobs_auto_router_shadow_eval_get"]; put?: never; @@ -838,20 +839,21 @@ export interface paths { put?: never; /** * Start Shadow Eval - * @description Start a shadow eval: duplicate a sampled slice of a key's live traffic against a second - * arm, judge the two responses blind, and stratify win rates by tier and by the model that - * served the real arm. + * @description Start a shadow eval: duplicate a sampled slice of one or more keys' live traffic against + * a second arm, judge the two responses blind, and stratify win rates by tier, by the model + * that served the real arm, and by key. * - * A forward job answers whether the key should adopt router_name: it samples the requests + * A forward job answers whether the keys should adopt router_name: it samples the requests * the router did not serve and duplicates them through it. A reverse job answers whether a * key already on the router still gains from it: it samples the requests the router did * serve and duplicates them against baseline_model. A key can hold one active job per * direction, so both questions can run at once. * - * Shadow responses are never served to users. The job samples until it has judged - * max_turns turns, reaches the end of its window, or is stopped; sampling changes - * propagate to pods within about 10 seconds. Shadow and judge calls bill to the - * shadowed key but are excluded from request counts and auto-router adoption metrics. + * Shadow responses are never served to users. Each key samples until it has judged + * max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one + * key running out of budget does not end sampling for the others; sampling changes + * propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed + * key but are excluded from request counts and auto-router adoption metrics. */ post: operations["start_shadow_eval_auto_router_shadow_eval_start_post"]; delete?: never; @@ -891,7 +893,12 @@ export interface paths { put?: never; /** * Stop Shadow Eval Job - * @description Stop an active shadow eval job. Attempts are kept; sampling halts within ~10s. + * @description Stop an active shadow eval job, every key it scopes at once. Attempts are kept; + * sampling halts within ~10s. Keys that already stopped on their own budget keep the + * stopped_at they earned. The statement is the whole state machine: it claims the job + * only while a leg still samples inside the window with no stop recorded, so a racing + * operator, a same-instant budget spend, and a repeat stop all read the same 400 with + * the status the job actually holds. */ post: operations["stop_shadow_eval_job_auto_router_shadow_eval__job_id__stop_post"]; delete?: never; @@ -27746,6 +27753,8 @@ export interface components { * @default 0 */ completion_tokens: number | null; + /** Created At */ + created_at?: string | null; /** Endtime */ endTime: string | null; /** Messages */ @@ -27789,6 +27798,8 @@ export interface components { * @default 0 */ total_tokens: number | null; + /** Updated At */ + updated_at?: string | null; /** * User * @default @@ -33197,18 +33208,49 @@ export interface components { timeout?: number | null; }; /** - * ShadowEvalJobResponse - * @description A shadow-eval job. Validates directly from the prisma record (job_id reads the - * row's id); status is derived from stopped_at and ends_at, never stored, so no writer - * anywhere can produce an inconsistent one. Aggregate fields are populated by the - * detail endpoint only and stay None on list responses. + * ShadowEvalJobKeyResponse + * @description One key a job shadows, with its own budget and stop state. */ - ShadowEvalJobResponse: { + ShadowEvalJobKeyResponse: { /** * Api Key Id - * @description The hashed virtual key whose traffic this job evaluates, and only that key's + * @description The hashed virtual key whose traffic this entry scopes */ api_key_id: string; + /** + * Attempt Count + * @description This key's sampled attempts so far, judged and errored alike, the same count the sampler budgets against max_turns; populated on list and detail responses. Frozen at stopped_at once the key is stamped, so in-flight attempts landing after a stop never reclassify it + */ + attempt_count?: number | null; + /** + * Key Alias + * @description Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted + */ + key_alias?: string | null; + /** + * Key Name + * @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias + */ + key_name?: string | null; + /** + * Max Turns + * @description This key's own sample budget, independent of its siblings' + */ + max_turns: number; + /** + * Stopped At + * @description When this key's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset + */ + stopped_at?: string | null; + }; + /** + * ShadowEvalJobResponse + * @description A shadow-eval job over one or more keys, each with its own budget and stop state; + * status is derived from stopped_by, the keys' stop and budget state, and ends_at, + * never stored, so no writer anywhere can produce an inconsistent one. Aggregate + * fields are populated by the detail endpoint only and stay None on list responses. + */ + ShadowEvalJobResponse: { /** Baseline Model */ baseline_model?: string | null; /** @@ -33247,22 +33289,15 @@ export interface components { */ judged_count?: number | null; /** - * Key Alias - * @description Alias of the shadowed key, resolved from the key row at read time; None when unset or deleted + * Keys + * @description The keys whose traffic this job evaluates, and only those keys', each with its own budget */ - key_alias?: string | null; - /** - * Key Name - * @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias - */ - key_name?: string | null; + keys: components["schemas"]["ShadowEvalJobKeyResponse"][]; /** * Last Error * @description Most recent attempt error; detail endpoint only */ last_error?: string | null; - /** Max Turns */ - max_turns: number; /** @description Stratified verdicts; detail endpoint only */ results?: components["schemas"]["ShadowEvalResult"] | null; /** Router Name */ @@ -33271,13 +33306,19 @@ export interface components { shadow_percentage: number; /** * Status - * @description A job whose window has passed reads completed even if a later sweep stamped - * stopped_at; stopped means sampling ended before the window did. + * @description Three recorded facts, no history-guessing: a stop is stopped_by (the migration + * backfills it for every job that displayed stopped when the column arrived, so the + * pre-column population is closed), completion is the window passing or every key + * spending its budget, and anything else is running. The all-keys-stamped fallback + * covers only stops written by pre-column pods during a rolling deploy. * @enum {string} */ readonly status: "running" | "completed" | "stopped"; - /** Stopped At */ - stopped_at?: string | null; + /** + * Stopped By + * @description The operator who stopped the job early, recorded by the stop endpoint; 'unknown' backfilled by migration for jobs that displayed stopped when the column arrived; None when the job ended on its own. Its presence is what makes a job read stopped rather than completed + */ + stopped_by?: string | null; }; /** * ShadowEvalResult @@ -33286,9 +33327,14 @@ export interface components { ShadowEvalResult: { /** * By Current Model - * @description Sliced by the model that served the real arm: the key's incumbent models in forward mode, and in reverse the models the router itself picked + * @description Sliced by the model that served the real arm: the keys' incumbent models in forward mode, and in reverse the models the router itself picked */ by_current_model: components["schemas"]["ShadowEvalSlice"][]; + /** + * By Key + * @description One slice per scoped key that has judged verdicts, grouped on the raw key hash. Keys the job scopes but has not judged a turn for yet are absent rather than reported as zero + */ + by_key: components["schemas"]["ShadowEvalSlice"][]; /** By Tier */ by_tier: components["schemas"]["ShadowEvalSlice"][]; /** Overall Shadow Win Rate Pct */ @@ -33496,14 +33542,14 @@ export interface components { }; /** * StartShadowEvalRequest - * @description Start duplicating a key's traffic for blind comparison against an auto-router. + * @description Start duplicating one or more keys' traffic for blind comparison against an auto-router. */ StartShadowEvalRequest: { /** - * Api Key Id - * @description The hashed virtual key whose traffic will be shadowed. Shadow evaluation runs ONLY on this key's traffic; requests made with any other key are not sampled. + * Api Key Ids + * @description The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these keys' traffic; requests made with any other key are not sampled. Each key carries its own max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 keys per job, which also bounds every read the job's endpoints make. */ - api_key_id: string; + api_key_ids: string[]; /** * Baseline Model * @description Required when direction is reverse and rejected otherwise: the fixed model the router's own responses are judged against. Must be a plain model rather than another auto-router @@ -33530,7 +33576,7 @@ export interface components { judge_model: string; /** * Max Turns - * @description Sample budget: the job judges at most this many turns, then completes. This is also the spend bound; expected judge cost is roughly max_turns times one judge call + * @description Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, so a job over N keys judges at most N times max_turns turns. This is also the spend bound; expected judge cost is roughly that turn ceiling times one judge call * @default 200 */ max_turns: number; @@ -37927,7 +37973,7 @@ export interface operations { list_shadow_eval_jobs_auto_router_shadow_eval_get: { parameters: { query?: { - /** @description Filter to jobs shadowing this key */ + /** @description Filter to jobs that shadow this key, alone or alongside others */ api_key_id?: string | null; /** @description Newest jobs to return */ limit?: number; diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts index cc4fe64e4c8..a38b96e74c2 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { describe, it, expect, beforeEach, vi } from "vitest"; import { clearTokenCookies, getCookie, storeLoginToken } from "./cookieUtils"; import { getToken, setToken } from "./mcpTokenStore"; diff --git a/ui/litellm-dashboard/src/utils/dataUtils.test.ts b/ui/litellm-dashboard/src/utils/dataUtils.test.ts index d975a4a73f4..f696780bc0a 100644 --- a/ui/litellm-dashboard/src/utils/dataUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/dataUtils.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { describe, it, expect, beforeEach, vi, afterEach } from "vitest"; import { copyToClipboard, formatNumberWithCommas, getSpendString, updateExistingKeys } from "./dataUtils"; diff --git a/ui/litellm-dashboard/src/utils/localStorageUtils.test.ts b/ui/litellm-dashboard/src/utils/localStorageUtils.test.ts index 934c358cb74..2ef5e1d5b43 100644 --- a/ui/litellm-dashboard/src/utils/localStorageUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/localStorageUtils.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { describe, it, expect, vi, beforeEach } from "vitest"; import { emitLocalStorageChange, diff --git a/ui/litellm-dashboard/src/utils/mcpTokenStore.test.ts b/ui/litellm-dashboard/src/utils/mcpTokenStore.test.ts index 0d096677f0d..e384f41042a 100644 --- a/ui/litellm-dashboard/src/utils/mcpTokenStore.test.ts +++ b/ui/litellm-dashboard/src/utils/mcpTokenStore.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { clearAllMcpTokens, getToken, isTokenValid, removeToken, setToken } from "./mcpTokenStore"; diff --git a/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts b/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts index 0f3a8b4bf69..3d1e83ad236 100644 --- a/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/returnUrlUtils.test.ts @@ -1,3 +1,5 @@ +// @vitest-environment jsdom + import { buildLoginUrlWithReturn, clearStoredReturnUrl, diff --git a/ui/litellm-dashboard/tests/setup.unit.ts b/ui/litellm-dashboard/tests/setup.unit.ts new file mode 100644 index 00000000000..4eb0c473989 --- /dev/null +++ b/ui/litellm-dashboard/tests/setup.unit.ts @@ -0,0 +1,12 @@ +import { vi } from "vitest"; + +vi.mock("@/lib/toast", () => ({ + toast: { + success: vi.fn(), + info: vi.fn(), + warning: vi.fn(), + error: vi.fn(), + fromError: vi.fn(), + dismiss: vi.fn(), + }, +})); diff --git a/ui/litellm-dashboard/vitest.config.ts b/ui/litellm-dashboard/vitest.config.ts index 7d8825909d8..ca2a25cba9f 100644 --- a/ui/litellm-dashboard/vitest.config.ts +++ b/ui/litellm-dashboard/vitest.config.ts @@ -12,17 +12,81 @@ const staticImageData: Plugin = { }, }; -const config: ViteUserConfig = { +const sharedViteConfig = { plugins: [staticImageData], + resolve: { alias: { "@": resolve(__dirname, "src") } }, + define: { "import.meta.vitest": "undefined" }, + esbuild: { jsx: "automatic", jsxImportSource: "react" } as const, +}; + +const TEST_TS_FILES_THAT_RENDER_REACT: readonly string[] = [ + "src/**/hooks/**/*.test.ts", + "src/**/cost-tracking/_components/**/use_*.test.ts", + "src/**/models-and-endpoints/detailNavigation.test.ts", + "src/**/models-and-endpoints/vertexCredentialsUpload.test.ts", + "src/components/chat/useChatHistory.test.ts", + "src/lib/forms/pickDirty.test.ts", +]; + +const jsdomTier = { + environment: "./tests/jsdomFetchEnv.ts", + setupFiles: ["tests/setupTests.ts"], + globals: true, + css: true, + testTimeout: 60_000, + hookTimeout: 30_000, +}; + +const config: ViteUserConfig = { + ...sharedViteConfig, test: { - environment: "./tests/jsdomFetchEnv.ts", - setupFiles: ["tests/setupTests.ts"], - globals: true, - css: true, // lets you import CSS/modules without extra mocks - testTimeout: 60000, - hookTimeout: 30000, + projects: [ + { + ...sharedViteConfig, + test: { + name: "unit", + environment: "node", + setupFiles: ["tests/setup.unit.ts"], + globals: true, + testTimeout: 60_000, + hookTimeout: 30_000, + include: ["src/**/*.test.ts", "tests/**/*.test.ts"], + exclude: ["node_modules/**", ...TEST_TS_FILES_THAT_RENDER_REACT], + }, + }, + { + ...sharedViteConfig, + test: { + ...jsdomTier, + name: "component", + include: ["src/**/*.test.tsx", "tests/**/*.test.tsx", ...TEST_TS_FILES_THAT_RENDER_REACT], + exclude: ["node_modules/**", "**/*.integration.test.tsx"], + }, + }, + { + ...sharedViteConfig, + test: { + ...jsdomTier, + name: "integration", + include: ["src/**/*.integration.test.tsx", "tests/**/*.integration.test.tsx"], + exclude: ["node_modules/**"], + }, + }, + { + ...sharedViteConfig, + test: { + name: "types", + include: [], + typecheck: { + enabled: true, + include: ["src/**/*.test-d.ts", "src/**/*.test-d.tsx"], + ignoreSourceErrors: true, + }, + }, + }, + ], silent: process.env.CI ? "passed-only" : false, - teardownTimeout: 60000, + teardownTimeout: 60_000, coverage: { provider: "v8", reporter: ["text", "lcov"], @@ -32,37 +96,16 @@ const config: ViteUserConfig = { "**/*.test.*", "**/*.test-d.*", "**/*.spec.*", - "tests/**", - "node_modules/**", ".next/**", "out/**", - "**/*.config.*", "postcss.config.*", "tailwind.config.*", "next.config.*", ], }, - exclude: ["node_modules/**"], - include: ["src/**/*.test.ts", "src/**/*.test.tsx", "tests/**/*.test.ts", "tests/**/*.test.tsx"], - typecheck: { - include: ["src/**/*.test-d.ts", "src/**/*.test-d.tsx"], - ignoreSourceErrors: true, - }, - }, - resolve: { - alias: { - "@": resolve(__dirname, "src"), - }, - }, - define: { - "import.meta.vitest": "undefined", - }, - esbuild: { - jsx: "automatic", - jsxImportSource: "react", }, };