From 2dcd45386045b44f5a61952094a3f14c9cbf504e Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 20 Aug 2026 14:55:21 -0700 Subject: [PATCH] feat(shadow_eval)!: gate the per-key budget on dollar spend instead of turns (#37555) --- .../migration.sql | 5 + .../litellm_proxy_extras/schema.prisma | 4 +- litellm/integrations/shadow_eval_logger.py | 133 +++++++-- .../auto_router_endpoints.py | 58 ++-- litellm/proxy/proxy_server.py | 7 + litellm/proxy/schema.prisma | 4 +- .../auto_router_endpoints.py | 55 +++- schema.prisma | 4 +- .../integrations/test_shadow_eval_logger.py | 258 ++++++++++++++++-- .../test_auto_router_endpoints.py | 139 +++++++++- .../_components/ShadowEvalSection.test.tsx | 52 ++-- .../_components/ShadowEvalSection.tsx | 57 ++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 33 ++- 13 files changed, 671 insertions(+), 138 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_shadow_eval_max_budget/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_shadow_eval_max_budget/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_shadow_eval_max_budget/migration.sql new file mode 100644 index 00000000000..7b60dca9415 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_shadow_eval_max_budget/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN "max_budget" DOUBLE PRECISION; + +-- AlterTable +ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN "shadow_cost" DOUBLE PRECISION NOT NULL DEFAULT 0; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 60058c777ca..d9959677116 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -1502,7 +1502,8 @@ model LiteLLM_ShadowEvalJob { baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // this key's sample budget: judge at most this many turns + max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise + max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets created_at DateTime @default(now()) created_by String? ends_at DateTime @@ -1525,6 +1526,7 @@ model LiteLLM_ShadowEvalAttempt { shadow_model String? confidence Float? judge_cost Float @default(0) + shadow_cost Float @default(0) error String? created_at DateTime @default(now()) diff --git a/litellm/integrations/shadow_eval_logger.py b/litellm/integrations/shadow_eval_logger.py index da02db4e44b..5f4e7c71395 100644 --- a/litellm/integrations/shadow_eval_logger.py +++ b/litellm/integrations/shadow_eval_logger.py @@ -10,7 +10,7 @@ import asyncio import hashlib import random import traceback -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from itertools import groupby @@ -42,8 +42,9 @@ if TYPE_CHECKING: from litellm.router import Router from litellm.types.utils import StandardLoggingPayload -# A job starting, stopping, or hitting its turn budget propagates to sampling within one -# TTL; the turn budget can overshoot by at most one TTL of in-flight samples per pod. +# A job starting, stopping, or hitting a budget propagates to sampling within one TTL; +# the spend gate re-checks the cross-pod counter at pipeline entry, so it overshoots +# only by the samples already in flight when the cap is crossed. _JOBS_CACHE_TTL_SECONDS: Final = 10 # Concurrent shadow+judge pipelines per pod: a traffic spike turns into skipped samples @@ -340,13 +341,24 @@ def _failure_detail(e: BaseException) -> str: return f"{type(e).__name__}{location}: {e}" -def _judge_call_cost(response: object) -> float: - """Price a judge call, treating an unmapped judge model as free rather than fatal.""" +def _call_cost(response: object) -> float: + """Price one eval-arm call with the figure the spend pipeline bills: the router client + stamps _hidden_params.response_cost from the deployment's own pricing, which the public + price map lookup below cannot see (it reads 0 for deployment-priced models).""" + getter: Final = getattr(getattr(response, "_hidden_params", None), "get", None) + stamped: Final = getter("response_cost") if callable(getter) else None + if isinstance(stamped, (int, float)): + return float(stamped) + return _price_map_cost(response) + + +def _price_map_cost(response: object) -> float: + """Public price map fallback, treating an unmapped model as free rather than fatal.""" import litellm try: return litellm.completion_cost(completion_response=response) or 0.0 - except Exception: # noqa: BLE001 # unmapped judge model: the verdict still counts, cost stays 0 + except Exception: # noqa: BLE001 # unmapped model: the attempt still counts, cost stays 0 return 0.0 @@ -374,6 +386,32 @@ def _judge_user_prompt(conversation: str, response_a: str, response_b: str) -> s ) +def _job_spend_counter_key(job_id: str) -> str: + return f"spend:shadow_eval:{job_id}" + + +async def _job_spend_from_counter(counter_key: str, fallback_spend: float, max_budget: float) -> float: + """The leg's spend through the cross-pod counter the key budget gates read. The owner + degrades internally to the fill-time DB floor and raises only under fail-closed + enforcement, which the caller honors by skipping the sample.""" + from litellm.proxy.proxy_server import get_current_spend + + return await get_current_spend(counter_key=counter_key, fallback_spend=fallback_spend, max_budget=max_budget) + + +async def _add_job_spend_to_counter(counter_key: str, cost: float) -> None: + """Advance the counter the moment a cost is known, so even a lost row closes the gate. + Known failure mode: a Redis outage freezes the counter (the owner invalidates it), the + gate degrades to the fill floor, and overshoot grows to in-flight plus one TTL of + samples, the same degradation the key budget counters accept.""" + try: + from litellm.proxy.proxy_server import increment_spend_counter + + await increment_spend_counter(counter_key=counter_key, increment=cost) + except Exception as e: # noqa: BLE001 # attempt recording must proceed; the row stays truth and the fill floor gates + verbose_logger.warning("shadow_eval: spend counter increment failed for %s: %s", counter_key, e) + + async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool: """Whether the shadowed key or its team is over budget, decided by the same owners the request path uses, so counter keys and thresholds can never drift from auth's. @@ -438,8 +476,8 @@ def _request_was_routed_by(request_metadata: Mapping[str, object], router_name: @dataclass(frozen=True, slots=True) class _CallFailure: - """A shadow or judge call that produced no usable response. cost carries any judge - spend the failed attempt still billed, so job-level judge_spend never undercounts.""" + """A shadow or judge call that produced no usable response. cost carries any spend + the failed call still billed, so job-level spend figures never undercount.""" error: str cost: float = 0.0 @@ -452,6 +490,7 @@ class _ShadowResponse: text: str model: str tier: str | None + cost: float @dataclass(frozen=True, slots=True) @@ -478,8 +517,10 @@ class ActiveShadowEvalJob(BaseModel): shadow_percentage: float judge_model: str max_turns: int + max_budget: float | None = None ends_at: datetime attempts: int = 0 + spend: float = 0.0 @field_validator("ends_at") @classmethod @@ -500,7 +541,7 @@ class ActiveShadowEvalJob(BaseModel): return self.baseline_model or self.router_name -def _as_active_job(record: object, attempts: int) -> ActiveShadowEvalJob | None: +def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None: """The sampling path's view of one job row, or None for a row it cannot sample: an unknown direction, or a reverse job with no baseline model to duplicate against. Failing closed here is what keeps the dispatch path total.""" @@ -509,7 +550,7 @@ def _as_active_job(record: object, attempts: int) -> ActiveShadowEvalJob | None: except ValidationError as e: verbose_logger.debug("shadow_eval: skipping unsamplable job row: %s", e) return None - return job.model_copy(update={"attempts": attempts}) + return job.model_copy(update={"attempts": attempts, "spend": spend}) # mutable-ok: pydantic update payload _jobs_cache: Final = InMemoryCache(max_size_in_memory=4, default_ttl=_JOBS_CACHE_TTL_SECONDS) @@ -524,12 +565,17 @@ class ShadowEvalLogger(CustomLogger): router_provider: Callable[[], "Router | None"] | None = None, prisma_provider: Callable[[], "PrismaClient | None"] | None = None, jobs_cache: InMemoryCache | None = None, + job_spend_reader: Callable[[str, float, float], Awaitable[float]] | None = None, + job_spend_writer: Callable[[str, float], Awaitable[None]] | None = None, ) -> None: """Providers are callables so the proxy's lazily-initialized globals are resolved - at call time, not at logger construction.""" + at call time, not at logger construction. The spend reader and writer wrap the + proxy's cross-pod spend counter; tests inject a plain in-memory pair.""" self._router_provider = router_provider or default_router_provider self._prisma_provider = prisma_provider or _default_prisma_provider self._jobs_cache = jobs_cache or _jobs_cache + self._read_job_spend = job_spend_reader or _job_spend_from_counter + self._write_job_spend = job_spend_writer or _add_job_spend_to_counter self._inflight_shadow_tasks: int = 0 # Starts per job since the last cache fill, never decremented within a # generation; the refill absorbs written rows and resets. @@ -556,18 +602,26 @@ class ShadowEvalLogger(CustomLogger): await prisma.db.litellm_shadowevalattempt.group_by( by=["job_id"], count=True, + sum={"judge_cost": True, "shadow_cost": True}, # mutable-ok: Prisma aggregate spec where={"job_id": {"in": [str(record.id) for record in records]}}, # mutable-ok: Prisma filter ) if records else () ) - attempt_counts: Final = {str(row["job_id"]): int(row["_count"]["_all"]) for row in grouped or []} + attempt_stats: Final = { # mutable-ok: frozen snapshot of the grouped read + str(row["job_id"]): ( + int(row["_count"]["_all"]), + float((row["_sum"] or {}).get("judge_cost") or 0.0) + + float((row["_sum"] or {}).get("shadow_cost") or 0.0), + ) + for row in grouped or [] + } by_key: Final = tuple( sorted( ( (str(record.api_key_id), job) for record in records or [] - if (job := _as_active_job(record, attempt_counts.get(str(record.id), 0))) is not None + if (job := _as_active_job(record, *attempt_stats.get(str(record.id), (0, 0.0)))) is not None ), key=itemgetter(0), ) @@ -624,6 +678,7 @@ class ShadowEvalLogger(CustomLogger): for job in (await self._active_jobs()).get(str(api_key_hash), ()) if datetime.now(timezone.utc) < job.ends_at and job.attempts + self._job_starts.get(job.id, 0) < job.max_turns + and (job.max_budget is None or job.spend < job.max_budget) and _sample_hits(request_id, job.id, job.shadow_percentage) and _request_was_routed_by(request_metadata, job.router_name) == (job.direction == "reverse") ) @@ -684,12 +739,28 @@ class ShadowEvalLogger(CustomLogger): return if await _key_or_team_is_over_budget(parent_metadata): return - + if job.max_budget is not None: + try: + spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget) + except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it + verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e) + return + if spend >= job.max_budget: + return shadow: Final = await self._call_router_shadow(job.shadow_target, messages, shadow_params, parent_metadata) - if isinstance(shadow, _CallFailure): - await self._record_attempt(prisma, job, request_id, control_tier, outcome="error", error=shadow.error) - return - + except Exception as e: # noqa: BLE001 # detached task: nothing billed yet, record and never raise + verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e) + await self._record_attempt( + prisma, job, request_id, control_tier, outcome="error", error=f"pipeline error: {e}" + ) + return + if isinstance(shadow, _CallFailure): + await self._record_attempt( + prisma, job, request_id, control_tier, outcome="error", error=shadow.error, shadow_cost=shadow.cost + ) + return + # From here the shadow call has billed, so every exit records its cost. + try: verdict: Final = await self._call_judge( judge_model=job.judge_model, messages=messages, @@ -707,6 +778,7 @@ class ShadowEvalLogger(CustomLogger): error=verdict.error, shadow=shadow, judge_cost=verdict.cost, + shadow_cost=shadow.cost, ) return await self._record_attempt( @@ -719,15 +791,23 @@ class ShadowEvalLogger(CustomLogger): real_model=real_model, confidence=verdict.confidence, judge_cost=verdict.cost, + shadow_cost=shadow.cost, ) - except Exception as e: # noqa: BLE001 # detached task: record what happened, never raise + except Exception as e: # noqa: BLE001 # detached task: the shadow call billed, record its cost, never raise verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e) await self._record_attempt( - prisma, job, request_id, control_tier, outcome="error", error=f"pipeline error: {e}" + prisma, + job, + request_id, + control_tier, + outcome="error", + error=f"pipeline error: {e}", + shadow=shadow, + shadow_cost=shadow.cost, ) - @staticmethod async def _record_attempt( + self, prisma: "PrismaClient | None", job: ActiveShadowEvalJob, request_id: str, @@ -738,8 +818,11 @@ class ShadowEvalLogger(CustomLogger): real_model: str = "", confidence: float | None = None, judge_cost: float = 0.0, + shadow_cost: float = 0.0, error: str | None = None, ) -> None: + if judge_cost + shadow_cost > 0: + await self._write_job_spend(_job_spend_counter_key(job.id), judge_cost + shadow_cost) if prisma is None: return try: @@ -753,6 +836,7 @@ class ShadowEvalLogger(CustomLogger): "shadow_model": shadow.model if shadow else None, "confidence": confidence, "judge_cost": judge_cost, + "shadow_cost": shadow_cost, "error": error[:_MAX_ERROR_CHARS] if error else None, } ) @@ -792,11 +876,12 @@ class ShadowEvalLogger(CustomLogger): return _CallFailure(f"shadow router call failed: {_failure_detail(e)}") text: Final = _chat_final_text(response) if not text: - return _CallFailure("shadow router returned an empty response") + return _CallFailure("shadow router returned an empty response", cost=_call_cost(response)) return _ShadowResponse( text=text, model=str(getattr(response, "model", None) or _routing_decision(shadow_metadata).get("routed_model") or ""), tier=_routed_tier(shadow_metadata), + cost=_call_cost(response), ) async def _call_judge( @@ -843,11 +928,11 @@ class ShadowEvalLogger(CustomLogger): verdict: Final = PairwiseVerdict.model_validate(parse_json_verdict(raw)) except Exception as e: # noqa: BLE001 # malformed verdicts become error rows verbose_logger.debug("shadow_eval: unparseable judge verdict: %s", e) - return _CallFailure(f"unparseable judge verdict: {e}", cost=_judge_call_cost(response)) + return _CallFailure(f"unparseable judge verdict: {e}", cost=_call_cost(response)) return _JudgeVerdict( preference=_unmask_preference(verdict.preference, real_is_a), confidence=max(0.0, min(1.0, verdict.confidence)), - cost=_judge_call_cost(response), + cost=_call_cost(response), ) diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index d47bd7fa311..46aac82473c 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -37,6 +37,7 @@ from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.team_repository import TeamRepository from litellm.router_strategy.complexity_router import ComplexityRouter from litellm.types.management_endpoints.auto_router_endpoints import ( + SHADOW_EVAL_TURN_VALVE, AutoRouterBenchmarkGroup, AutoRouterBenchmarksResponse, AutoRouterBenchmarkTotals, @@ -662,12 +663,19 @@ _ATTEMPT_AGG_BY_TIER_SQL: Final = "SELECT COALESCE(tier, 'UNCLASSIFIED') AS grp, _ATTEMPT_AGG_BY_MODEL_SQL: Final = "SELECT COALESCE(real_model, 'unknown') AS grp," + _ATTEMPT_AGG_SELECT _ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT +# These guards derive spend from attempt rows, the cross-pod authority; the sampler also +# reads the live counter, so admission can stop before a row-based guard would fire (safe +# direction, and mid-deploy rows from old pods price as judge-only until the deploy ends). _SWEEP_FINISHED_JOBS_SQL: Final = """ UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = (NOW() AT TIME ZONE 'utc') WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL AND ( j.ends_at <= (NOW() AT TIME ZONE 'utc') OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns + OR ( + j.max_budget IS NOT NULL + AND (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_budget + ) ) """ @@ -681,7 +689,7 @@ WHERE job_id = ANY($1::text[]) """ _ATTEMPT_COUNTS_SQL: Final = """ -SELECT a.job_id, COUNT(*)::int AS attempt_count +SELECT a.job_id, COUNT(*)::int AS attempt_count, COALESCE(SUM(a.judge_cost + a.shadow_cost), 0)::float AS spend FROM "LiteLLM_ShadowEvalAttempt" a JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id WHERE a.job_id = ANY($1::text[]) AND (j.stopped_at IS NULL OR a.created_at <= j.stopped_at) @@ -697,6 +705,10 @@ WHERE group_id = $1 AND stopped_by IS NULL SELECT 1 FROM "LiteLLM_ShadowEvalJob" k WHERE k.group_id = $1 AND k.stopped_at IS NULL AND (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_turns + AND ( + k.max_budget IS NULL + OR (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_budget + ) ) """ @@ -704,6 +716,7 @@ WHERE group_id = $1 AND stopped_by IS NULL class _AttemptCountRow(BaseModel): job_id: str attempt_count: int + spend: float _ATTEMPT_COUNT_ROWS: Final = TypeAdapter(list[_AttemptCountRow]) @@ -770,6 +783,7 @@ class _LegRow(BaseModel): judge_model: str shadow_percentage: float max_turns: int + max_budget: float | None = None created_at: datetime ends_at: datetime stopped_at: datetime | None = None @@ -789,22 +803,25 @@ class _LegRow(BaseModel): _LEG_ROWS: Final = TypeAdapter(list[_LegRow]) -async def _leg_attempt_counts(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> Mapping[str, int]: - """Each leg's attempt count by leg id, judged and errored alike, in one grouped read. - It is the same count the sampler budgets against max_turns, so the derived status - flips to completed exactly when sampling actually ends. A stamped leg's count freezes - at its stopped_at: in-flight attempts that land after the stamp are excluded, so they - can never reclassify a leg that was stopped under budget as budget-spent.""" +async def _leg_attempt_counts(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> Mapping[str, _AttemptCountRow]: + """Each leg's attempt count and recorded spend by leg id, judged and errored alike, in + one grouped read. They are the same figures the sampler budgets against max_turns and + max_budget, so the derived status flips to completed exactly when sampling actually + ends. A stamped leg's figures freeze at its stopped_at: in-flight attempts that land + after the stamp are excluded, so they can never reclassify a leg that was stopped + under budget as budget-spent.""" if not legs: return MappingProxyType({}) rows: Final = _ATTEMPT_COUNT_ROWS.validate_python( await _query_raw(prisma_client, _ATTEMPT_COUNTS_SQL, [leg.id for leg in legs]) # mutable-ok: query param or () ) - return MappingProxyType({row.job_id: row.attempt_count for row in rows}) + return MappingProxyType({row.job_id: row for row in rows}) -def _group_response(group_id: str, legs: Sequence[_LegRow], attempt_counts: Mapping[str, int]) -> ShadowEvalJobResponse: +def _group_response( + group_id: str, legs: Sequence[_LegRow], attempt_counts: Mapping[str, _AttemptCountRow] +) -> ShadowEvalJobResponse: """The one constructor of a job response: the caller names the group and passes that group's legs. Config is read off the first leg because every leg carries the same copy, written by one create_many. No caller may serialize a raw row (that would leak a leg id @@ -816,8 +833,10 @@ def _group_response(group_id: str, legs: Sequence[_LegRow], attempt_counts: Mapp ShadowEvalJobKeyResponse( api_key_id=leg.api_key_id, max_turns=leg.max_turns, + max_budget=leg.max_budget, stopped_at=leg.stopped_at, - attempt_count=attempt_counts.get(leg.id, 0), + attempt_count=stats.attempt_count if (stats := attempt_counts.get(leg.id)) else 0, + spend=round(stats.spend, 6) if stats else 0.0, ) for leg in sorted(legs, key=lambda leg: leg.api_key_id) ), @@ -923,11 +942,12 @@ async def start_shadow_eval( serve and duplicates them against baseline_model. A key can hold one active job per direction, so both questions can run at once. - Shadow responses are never served to users. Each key samples until it has judged - max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one - key running out of budget does not end sampling for the others; sampling changes - propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed - key but are excluded from request counts and auto-router adoption metrics. + Shadow responses are never served to users. Each key samples until its recorded eval + spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's + window ends, or the job is stopped, so one key running out of budget does not end + sampling for the others; sampling changes propagate to pods within about 10 seconds. + Shadow and judge calls bill to the shadowed key but are excluded from request counts + and auto-router adoption metrics. """ from litellm.proxy.proxy_server import llm_router, prisma_client @@ -952,7 +972,7 @@ async def start_shadow_eval( ), ) - # A job whose window passed or whose turn budget ran out stopped sampling on its own, + # A job whose window passed or whose budget ran out stopped sampling on its own, # but its legs still hold their slots in the per-key, per-direction partial unique index # until stamped; free them so a new eval can start. Sweeping both directions is deliberate. requested: Final = list(data.api_key_ids) # mutable-ok: query param @@ -983,7 +1003,8 @@ async def start_shadow_eval( "baseline_model": data.baseline_model, "judge_model": data.judge_model, "shadow_percentage": data.shadow_percentage, - "max_turns": data.max_turns, + "max_turns": SHADOW_EVAL_TURN_VALVE, + "max_budget": data.max_budget, "created_by": user_api_key_dict.user_id, "created_at": now, "ends_at": ends_at, @@ -1007,7 +1028,8 @@ async def start_shadow_eval( keys=tuple( ShadowEvalJobKeyResponse( api_key_id=api_key_id, - max_turns=data.max_turns, + max_turns=SHADOW_EVAL_TURN_VALVE, + max_budget=data.max_budget, key_alias=labels[api_key_id].key_alias, key_name=labels[api_key_id].key_name, ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 12d0f13ebdd..b94a663fc15 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2994,6 +2994,13 @@ async def _is_spend_counter_cache_warm(counter_key: str) -> bool: return spend_counter_cache.in_memory_cache.get_cache(key=counter_key) is not None +async def increment_spend_counter(counter_key: str, increment: float): + """Public raw-counter increment for budget domains outside the entity scopes (e.g. + shadow eval's per-leg spend), sharing the primitive the entity counters use so + invalidation and read semantics can never drift.""" + return await _increment_spend_counter_cache(counter_key=counter_key, increment=increment) + + async def _increment_spend_counter_cache(counter_key: str, increment: float): if spend_counter_cache.redis_cache is not None: try: diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 60058c777ca..d9959677116 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -1502,7 +1502,8 @@ model LiteLLM_ShadowEvalJob { baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // this key's sample budget: judge at most this many turns + max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise + max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets created_at DateTime @default(now()) created_by String? ends_at DateTime @@ -1525,6 +1526,7 @@ model LiteLLM_ShadowEvalAttempt { shadow_model String? confidence Float? judge_cost Float @default(0) + shadow_cost Float @default(0) error String? created_at DateTime @default(now()) diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index d68d1dc9625..e2469d4c78f 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -169,6 +169,10 @@ ShadowEvalDirection: TypeAlias = Literal["forward", "reverse"] DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5" +# Sample-count ceiling written on every new job: a zero-cost error loop (a shadow arm that +# fails before billing) never consumes spend budget, so it must terminate on count instead. +SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000 + class StartShadowEvalRequest(BaseModel): """Start duplicating one or more keys' traffic for blind comparison against an auto-router.""" @@ -179,7 +183,7 @@ class StartShadowEvalRequest(BaseModel): description=( "The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these " "keys' traffic; requests made with any other key are not sampled. Each key carries its own " - "max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 " + "max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 " "keys per job, which also bounds every read the job's endpoints make." ), ) @@ -219,17 +223,27 @@ class StartShadowEvalRequest(BaseModel): le=30, description="How many days the job samples traffic before completing on its own", ) - max_turns: int = Field( - default=200, - ge=1, - le=2000, + max_budget: float = Field( + default=10.0, + ge=0.01, + le=10_000, description=( - "Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, " - "so a job over N keys judges at most N times max_turns turns. This is also the spend bound; " - "expected judge cost is roughly that turn ceiling times one judge call" + "Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with " + "the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval " + "spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight " + "samples can overshoot the cap by one sampling cache window" ), ) + @model_validator(mode="before") + @classmethod + def _reject_the_retired_turn_budget(cls, values: object) -> object: + """Pydantic ignores unknown fields, so a caller still sending max_turns would + silently run on the default dollar budget instead of the bound they asked for.""" + if isinstance(values, Mapping) and "max_turns" in values: + raise ValueError("max_turns was replaced by max_budget, the per-key USD cap on the eval's own spend") + return values + @field_validator("shadow_percentage") @classmethod def _round_percentage(cls, value: float) -> float: @@ -296,7 +310,19 @@ class ShadowEvalJobKeyResponse(BaseModel): """One key a job shadows, with its own budget and stop state.""" api_key_id: str = Field(description="The hashed virtual key whose traffic this entry scopes") - max_turns: int = Field(description="This key's own sample budget, independent of its siblings'") + max_turns: int = Field( + description=( + "This key's sample-count ceiling: the whole budget for jobs created before max_budget " + "existed, and the error-loop safety valve otherwise" + ) + ) + max_budget: float | None = Field( + default=None, + description=( + "This key's own USD budget for the eval's shadow and judge spend, independent of its " + "siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds" + ), + ) stopped_at: datetime | None = Field( default=None, description=( @@ -313,10 +339,19 @@ class ShadowEvalJobKeyResponse(BaseModel): "once the key is stamped, so in-flight attempts landing after a stop never reclassify it" ), ) + spend: float | None = Field( + default=None, + description=( + "This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets " + "against max_budget; populated on list and detail responses and frozen at stopped_at " + "exactly like attempt_count" + ), + ) @property def budget_spent(self) -> bool: - return self.attempt_count is not None and self.attempt_count >= self.max_turns + over_spend: Final = self.max_budget is not None and self.spend is not None and self.spend >= self.max_budget + return over_spend or (self.attempt_count is not None and self.attempt_count >= self.max_turns) key_alias: str | None = Field( default=None, diff --git a/schema.prisma b/schema.prisma index 60058c777ca..d9959677116 100644 --- a/schema.prisma +++ b/schema.prisma @@ -1502,7 +1502,8 @@ model LiteLLM_ShadowEvalJob { baseline_model String? // reverse only: the fixed model the router is judged against judge_model String shadow_percentage Float - max_turns Int // this key's sample budget: judge at most this many turns + max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise + max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets created_at DateTime @default(now()) created_by String? ends_at DateTime @@ -1525,6 +1526,7 @@ model LiteLLM_ShadowEvalAttempt { shadow_model String? confidence Float? judge_cost Float @default(0) + shadow_cost Float @default(0) error String? created_at DateTime @default(now()) diff --git a/tests/test_litellm/integrations/test_shadow_eval_logger.py b/tests/test_litellm/integrations/test_shadow_eval_logger.py index 514d5c6adca..4f6fea7b710 100644 --- a/tests/test_litellm/integrations/test_shadow_eval_logger.py +++ b/tests/test_litellm/integrations/test_shadow_eval_logger.py @@ -40,11 +40,19 @@ def _job(**overrides) -> ActiveShadowEvalJob: return ActiveShadowEvalJob(**{**defaults, **overrides}) -def _prisma(jobs=(), attempt_counts=()) -> MagicMock: +def _prisma(jobs=(), attempt_counts=(), attempt_costs=()) -> MagicMock: + costs = {job_id: {"judge_cost": judge, "shadow_cost": shadow} for job_id, judge, shadow in attempt_costs} prisma = MagicMock() prisma.db.litellm_shadowevaljob.find_many = AsyncMock(return_value=list(jobs)) prisma.db.litellm_shadowevalattempt.group_by = AsyncMock( - return_value=[{"job_id": job_id, "_count": {"_all": count}} for job_id, count in attempt_counts] + return_value=[ + { + "job_id": job_id, + "_count": {"_all": count}, + "_sum": costs.get(job_id, {"judge_cost": 0.0, "shadow_cost": 0.0}), + } + for job_id, count in attempt_counts + ] ) prisma.db.litellm_shadowevalattempt.create = AsyncMock() return prisma @@ -61,6 +69,7 @@ def _job_record(job: ActiveShadowEvalJob, api_key_id="key-hash") -> MagicMock: shadow_percentage=job.shadow_percentage, judge_model=job.judge_model, max_turns=job.max_turns, + max_budget=job.max_budget, ends_at=job.ends_at, ).items(): setattr(record, field, value) @@ -92,13 +101,32 @@ def _router(shadow_text="shadow answer", judge_json='{"preference": "A", "confid return router -def _logger(router=None, prisma=None, jobs=()) -> ShadowEvalLogger: +def _spend_counter(store=None): + """In-memory stand-in for the proxy's cross-pod spend counter: reads take the max of + the counter and the caller's fallback, exactly like get_current_spend does for a key + shape the reseed helpers do not know.""" + counter = store if store is not None else {} + + async def read(key, fallback_spend, max_budget): + return max(counter.get(key, 0.0), fallback_spend) + + async def write(key, cost): + counter[key] = counter.get(key, 0.0) + cost + + return counter, read, write + + +def _logger(router=None, prisma=None, jobs=(), counter_store=None) -> ShadowEvalLogger: cache = InMemoryCache(max_size_in_memory=4, default_ttl=60) + counter, read, write = _spend_counter(counter_store) logger = ShadowEvalLogger( router_provider=lambda: router, prisma_provider=lambda: prisma, jobs_cache=cache, + job_spend_reader=read, + job_spend_writer=write, ) + logger._test_counter = counter if jobs: cache.set_cache("shadow_eval:active_jobs", {"key-hash": tuple(jobs)}) return logger @@ -447,7 +475,9 @@ def test_failure_detail_names_the_raising_frame(): except TypeError as e: detail = _failure_detail(e) lineno = e.__traceback__.tb_lineno - assert detail == f"TypeError at test_shadow_eval_logger.py:{lineno}: 'tuple' object does not support item assignment" + assert ( + detail == f"TypeError at test_shadow_eval_logger.py:{lineno}: 'tuple' object does not support item assignment" + ) try: raise ValueError("p" * 5 * _MAX_ERROR_CHARS) @@ -456,6 +486,73 @@ def test_failure_detail_names_the_raising_frame(): assert "ValueError at test_shadow_eval_logger.py:" in truncated_row_error +def test_call_cost_prefers_the_billed_figure_over_the_public_price_map(monkeypatch): + """The router client stamps _hidden_params.response_cost from the deployment's own + pricing; the public map reads 0 for deployment-priced models, so budgets gated on it + would never close. The map is only the fallback for responses with no stamp.""" + import litellm as litellm_module + from litellm.integrations.shadow_eval_logger import _call_cost + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005) + stamped = MagicMock() + stamped._hidden_params = {"response_cost": 0.04} + assert _call_cost(stamped) == 0.04 + + from litellm.types.utils import HiddenParams + + object_stamped = MagicMock() + object_stamped._hidden_params = HiddenParams(response_cost=0.03) + assert _call_cost(object_stamped) == 0.03 + + unstamped = MagicMock() + unstamped._hidden_params = {"response_cost": None} + assert _call_cost(unstamped) == 0.005 + assert _call_cost({"choices": []}) == 0.005 + + +@pytest.mark.asyncio +async def test_a_cold_or_reset_counter_degrades_to_the_fill_floor_not_zero(monkeypatch: pytest.MonkeyPatch): + """The design leans on one owner contract: for a spend:shadow_eval:* key (no DB + reseed by design), get_current_spend returns the caller's fill-sum fallback whenever + the counter reads lower. A reset counter therefore degrades to the <=10s-stale DB + sum, never to zero, so a Redis expiry cannot re-open a spent budget by a full cap.""" + from litellm.proxy import proxy_server + + counter_key = "spend:shadow_eval:job-cold-test" + monkeypatch.setattr(proxy_server, "prisma_client", None) + proxy_server.spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.05) + try: + assert ( + await proxy_server.get_current_spend(counter_key=counter_key, fallback_spend=0.42, max_budget=1.0) == 0.42 + ) + proxy_server.spend_counter_cache.in_memory_cache.delete_cache(key=counter_key) + assert ( + await proxy_server.get_current_spend(counter_key=counter_key, fallback_spend=0.42, max_budget=1.0) == 0.42 + ) + finally: + proxy_server.spend_counter_cache.in_memory_cache.delete_cache(key=counter_key) + + +@pytest.mark.asyncio +async def test_an_unverifiable_budget_skips_the_sample_instead_of_spending(): + """A raising spend read (fail-closed enforcement, or an owner bug) must skip the + sample before any provider call, never admit it on a guess.""" + + async def unverifiable(key, fallback_spend, max_budget): + raise RuntimeError("budget unverifiable") + + prisma = _prisma() + router = _router() + logger = _logger(router=router, prisma=prisma, jobs=(_job(max_budget=1.0),)) + logger._read_job_spend = unverifiable + + await logger.async_log_success_event(_success_kwargs(), RESPONSE, None, None) + await _drain(logger) + + router.acompletion.assert_not_called() + prisma.db.litellm_shadowevalattempt.create.assert_not_called() + + def test_judge_prompt_is_bounded_however_large_the_inputs(): prompt = _judge_user_prompt("c" * 200_000, "a" * 200_000, "b" * 200_000) assert len(prompt) < _MAX_JUDGE_PROMPT_CHARS + 100 @@ -491,6 +588,7 @@ class TestSuccessHookSkipChain: assert row["shadow_model"] == "cheap-model" assert row["confidence"] == 0.9 assert row["judge_cost"] == 0.005 + assert row["shadow_cost"] == 0.005 assert row["error"] is None assert prisma.db.litellm_shadowevaljob.find_many.await_count == 0 @@ -580,6 +678,7 @@ class TestSuccessHookSkipChain: ({}, {"ends_at": datetime.now(timezone.utc) - timedelta(seconds=1)}), ({}, {"attempts": 200}), ({}, {"attempts": 199, "max_turns": 200, "_starts": 1}), + ({}, {"max_budget": 0.10, "spend": 0.10}), ], ids=[ "internal-origin", @@ -590,6 +689,7 @@ class TestSuccessHookSkipChain: "past-end", "turn-budget-reached", "budget-consumed-by-started-tasks", + "spend-budget-reached", ], ) async def test_skip_paths_store_nothing(self, kwargs_mutation, job_mutation): @@ -617,6 +717,61 @@ class TestSuccessHookSkipChain: assert prisma.db.litellm_shadowevalattempt.create.await_count == 1 + async def test_completed_pipelines_hold_spend_budget_within_a_cache_generation(self, monkeypatch): + """An attempt's recorded cost lands in the spend counter immediately, so the + second sample is skipped before any provider call even though the cached fill + still reads spend 0.""" + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005) + prisma = _prisma() + logger = _logger(router=_router(), prisma=prisma, jobs=(_job(max_budget=0.009, spend=0.0),)) + + await logger.async_log_success_event(_success_kwargs(request_id="req-1"), RESPONSE, None, None) + await _drain(logger) + await logger.async_log_success_event(_success_kwargs(request_id="req-2"), RESPONSE, None, None) + await _drain(logger) + + assert prisma.db.litellm_shadowevalattempt.create.await_count == 1 + assert logger._test_counter["spend:shadow_eval:job-1"] == 0.01 + + async def test_a_sibling_pod_sees_spend_through_the_shared_counter(self, monkeypatch): + """Two pods share the cross-pod counter: once pod A's attempts spend the budget, + pod B skips before its shadow call even though pod B's cached fill reads 0.""" + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005) + shared = {} + prisma_a = _prisma() + pod_a = _logger( + router=_router(), prisma=prisma_a, jobs=(_job(max_budget=0.009, spend=0.0),), counter_store=shared + ) + router_b = _router() + prisma_b = _prisma() + pod_b = _logger( + router=router_b, prisma=prisma_b, jobs=(_job(max_budget=0.009, spend=0.0),), counter_store=shared + ) + + await pod_a.async_log_success_event(_success_kwargs(request_id="req-1"), RESPONSE, None, None) + await _drain(pod_a) + await pod_b.async_log_success_event(_success_kwargs(request_id="req-2"), RESPONSE, None, None) + await _drain(pod_b) + + assert prisma_a.db.litellm_shadowevalattempt.create.await_count == 1 + prisma_b.db.litellm_shadowevalattempt.create.assert_not_called() + router_b.acompletion.assert_not_called() + + async def test_legacy_jobs_without_a_spend_budget_sample_on_turns_alone(self): + """A pre-migration job carries max_budget None: recorded spend must never gate it, + only its own max_turns can.""" + prisma = _prisma() + logger = _logger(router=_router(), prisma=prisma, jobs=(_job(max_budget=None, spend=999.0, attempts=5),)) + + await logger.async_log_success_event(_success_kwargs(request_id="req-1"), RESPONSE, None, None) + await _drain(logger) + + assert prisma.db.litellm_shadowevalattempt.create.await_count == 1 + async def test_v1_messages_surface_forwards_identity_from_litellm_metadata(self): """/v1/messages stores identity in litellm_params.litellm_metadata, so the hook resolves the bucket through the shared helper; every surface forwards the same @@ -714,7 +869,7 @@ class TestActiveJobsCache: async def test_cache_refill_resets_the_starts_counter(self): job = _job() - prisma = _prisma(jobs=[_job_record(job)], attempt_counts=[("job-1", 7)]) + prisma = _prisma(jobs=[_job_record(job)], attempt_counts=[("job-1", 7)], attempt_costs=[("job-1", 0.02, 0.03)]) logger = ShadowEvalLogger( router_provider=lambda: None, prisma_provider=lambda: prisma, @@ -722,9 +877,11 @@ class TestActiveJobsCache: ) logger._job_starts = {"job-1": 5} - await logger._active_jobs() + jobs = await logger._active_jobs() assert logger._job_starts == {} + assert jobs["key-hash"][0].attempts == 7 + assert jobs["key-hash"][0].spend == 0.05 @pytest.mark.asyncio @@ -749,9 +906,9 @@ class TestShadowPipeline: async def test_over_budget_key_skips_before_any_call(self, monkeypatch: pytest.MonkeyPatch): """The gate delegates to the auth path's own budget owner, so an over-budget verdict there (BudgetExceededError) skips the shadow before any provider call.""" - import litellm.proxy.auth.auth_checks as auth_checks from litellm.exceptions import BudgetExceededError from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth import auth_checks monkeypatch.setattr( auth_checks, @@ -777,13 +934,18 @@ class TestShadowPipeline: prisma.db.litellm_shadowevalattempt.create.assert_not_called() @pytest.mark.parametrize( - "router_factory,expected_error,expected_cost", + "router_factory,expected_error,expected_cost,expected_shadow_cost", [ - (lambda: _failing_router(), "provider exploded", 0.0), - (lambda: _router(judge_json="I prefer response A, definitely"), "unparseable judge verdict", 0.007), - (lambda: _router(judge_json='{"preference": "'), "unparseable judge verdict", 0.007), - (lambda: _router(judge_json="{}"), "unparseable judge verdict", 0.007), - (lambda: _router(judge_json='{"preference": "A", "confidence": "0.8'), "unparseable judge verdict", 0.007), + (lambda: _failing_router(), "provider exploded", 0.0, 0.0), + (lambda: _router(judge_json="I prefer response A, definitely"), "unparseable judge verdict", 0.007, 0.007), + (lambda: _router(judge_json='{"preference": "'), "unparseable judge verdict", 0.007, 0.007), + (lambda: _router(judge_json="{}"), "unparseable judge verdict", 0.007, 0.007), + ( + lambda: _router(judge_json='{"preference": "A", "confidence": "0.8'), + "unparseable judge verdict", + 0.007, + 0.007, + ), ], ids=[ "shadow-call-fails", @@ -794,7 +956,7 @@ class TestShadowPipeline: ], ) async def test_failures_become_error_rows_and_keep_billed_judge_cost( - self, router_factory, expected_error, expected_cost, monkeypatch: pytest.MonkeyPatch + self, router_factory, expected_error, expected_cost, expected_shadow_cost, monkeypatch: pytest.MonkeyPatch ): import litellm as litellm_module @@ -818,6 +980,66 @@ class TestShadowPipeline: assert expected_error in row["error"] assert row["confidence"] is None assert row["judge_cost"] == expected_cost + assert row["shadow_cost"] == expected_shadow_cost + + async def test_an_empty_shadow_reply_still_bills_its_cost(self, monkeypatch: pytest.MonkeyPatch): + """A shadow call that returns no extractable text has still billed; pricing it at + zero would keep the dollar gate open while shadow calls keep charging the key.""" + import litellm as litellm_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007) + prisma = _prisma() + logger = _logger(router=_router(shadow_text=""), prisma=prisma) + + await logger._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"] + assert row["outcome"] == "error" + assert "empty response" in row["error"] + assert row["shadow_cost"] == 0.007 + assert logger._test_counter["spend:shadow_eval:job-1"] == 0.007 + + async def test_a_pipeline_error_after_the_shadow_call_keeps_its_billed_cost(self, monkeypatch: pytest.MonkeyPatch): + """An unexpected error between the billed shadow call and the attempt write must + still record the shadow cost, or the per-key dollar gate undercounts forever.""" + import litellm as litellm_module + import litellm.integrations.shadow_eval_logger as shadow_eval_module + + monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007) + + def explode(conversation, response_a, response_b): + raise RuntimeError("judge prompt build failed") + + monkeypatch.setattr(shadow_eval_module, "_judge_user_prompt", explode) + prisma = _prisma() + logger = _logger(router=_router(), prisma=prisma) + + await logger._run_shadow_eval( + job=_job(), + request_id="req-1", + messages=({"role": "user", "content": "hi"},), + real_text="real answer", + real_model="claude-opus", + control_tier=None, + shadow_params={}, + parent_metadata={}, + ) + + row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"] + assert row["outcome"] == "error" + assert "pipeline error" in row["error"] + assert row["shadow_cost"] == 0.007 + assert row["judge_cost"] == 0.0 + assert logger._test_counter["spend:shadow_eval:job-1"] == 0.007 async def test_sub_calls_carry_identity_and_origin_but_never_parent_request_state(self): prisma = _prisma() @@ -918,9 +1140,7 @@ class TestDirection: router = _router() logger = _logger(router=router, prisma=prisma, jobs=(_reverse_job(),)) - await logger.async_log_success_event( - _success_kwargs(request_metadata=_routed_by()), RESPONSE, None, None - ) + await logger.async_log_success_event(_success_kwargs(request_metadata=_routed_by()), RESPONSE, None, None) await _drain(logger) assert router.acompletion.call_args_list[0].kwargs["model"] == "baseline-model" @@ -967,9 +1187,7 @@ class TestDirection: jobs=(_job(id="forward-job", router_name="other-router"), _reverse_job(id="reverse-job")), ) - await logger.async_log_success_event( - _success_kwargs(request_metadata=_routed_by()), RESPONSE, None, None - ) + await logger.async_log_success_event(_success_kwargs(request_metadata=_routed_by()), RESPONSE, None, None) await _drain(logger) rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.call_args_list] diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index c901696e108..6165e869989 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -490,7 +490,7 @@ from litellm.proxy.management_endpoints.auto_router_endpoints import ( start_shadow_eval, stop_shadow_eval_job, ) -from litellm.types.management_endpoints.auto_router_endpoints import StartShadowEvalRequest +from litellm.types.management_endpoints.auto_router_endpoints import SHADOW_EVAL_TURN_VALVE, StartShadowEvalRequest VIEWER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, api_key="sk-view", user_id="viewer") NON_ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-user", user_id="user") @@ -520,6 +520,7 @@ def _leg_record(**overrides: object) -> MagicMock: "judge_model": "anthropic/claude-sonnet-5", "shadow_percentage": 10.0, "max_turns": 200, + "max_budget": None, "created_at": datetime(2026, 8, 11, tzinfo=timezone.utc), "ends_at": datetime.now(timezone.utc) + timedelta(days=7), "stopped_at": None, @@ -549,11 +550,18 @@ def _shadow_prisma(legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-ha group read that matched on a leg id would come back empty.""" prisma = MagicMock() prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record(token) for token in known_keys]) + async def execute_raw(sql: str, *params: object): if "SET stopped_by" in sql: group = [row for row in stored if row.group_id == params[0]] counts = {row["job_id"]: row["attempt_count"] for row in prisma.attempt_rows} - sampling = any(row.stopped_at is None and counts.get(row.id, 0) < row.max_turns for row in group) + spends = {row["job_id"]: row["spend"] for row in prisma.attempt_rows} + sampling = any( + row.stopped_at is None + and counts.get(row.id, 0) < row.max_turns + and (row.max_budget is None or spends.get(row.id, 0.0) < row.max_budget) + for row in group + ) window_open = bool(group) and group[0].ends_at > datetime.now(timezone.utc) claimable = [row for row in group if row.stopped_by is None] if not (claimable and sampling and window_open): @@ -602,6 +610,7 @@ def _shadow_prisma(legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-ha "judge_model", "shadow_percentage", "max_turns", + "max_budget", "created_at", "ends_at", "stopped_at", @@ -639,7 +648,7 @@ def _start_request(**overrides: object) -> StartShadowEvalRequest: "shadow_percentage": 10.0, "judge_model": "anthropic/claude-sonnet-5", "duration_days": 7, - "max_turns": 200, + "max_budget": 5.0, } payload.update(overrides) return StartShadowEvalRequest.model_validate(payload) @@ -663,6 +672,9 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp assert "j.ends_at <= (NOW() AT TIME ZONE 'utc')" in sweep_sql assert "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in sweep_sql assert ">= j.max_turns" in sweep_sql + assert "j.max_budget IS NOT NULL" in sweep_sql + assert ">= j.max_budget" in sweep_sql + assert "SUM(a.judge_cost + a.shadow_cost)" in sweep_sql assert "j.api_key_id = ANY($1::text[])" in sweep_sql assert sweep_keys == ["key-hash", "key-hash-2"] prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once() @@ -670,15 +682,17 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp assert [row["api_key_id"] for row in rows] == ["key-hash", "key-hash-2"] assert len({frozenset((k, v) for k, v in row.items() if k != "api_key_id") for row in rows}) == 1 assert len({row["group_id"] for row in rows}) == 1 - assert all(row["max_turns"] == 200 and row["created_by"] == "admin" for row in rows) + assert all(row["max_turns"] == SHADOW_EVAL_TURN_VALVE and row["created_by"] == "admin" for row in rows) + assert all(row["max_budget"] == 5.0 for row in rows) assert all("status" not in row and "id" not in row for row in rows) assert response.job_id == rows[0]["group_id"] assert response.status == "running" assert response.judged_count is None - assert [(key.api_key_id, key.max_turns, key.key_alias) for key in response.keys] == [ - ("key-hash", 200, "prod-alpha"), - ("key-hash-2", 200, "prod-alpha"), + assert [(key.api_key_id, key.max_budget, key.key_alias) for key in response.keys] == [ + ("key-hash", 5.0, "prod-alpha"), + ("key-hash-2", 5.0, "prod-alpha"), ] + assert all(key.max_turns == SHADOW_EVAL_TURN_VALVE for key in response.keys) @pytest.mark.asyncio @@ -1041,10 +1055,10 @@ async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch ] ) prisma.attempt_rows = [ - {"job_id": "leg-1", "attempt_count": 5}, - {"job_id": "leg-2", "attempt_count": 6}, - {"job_id": "leg-3", "attempt_count": 5}, - {"job_id": "leg-4", "attempt_count": 3}, + {"job_id": "leg-1", "attempt_count": 5, "spend": 0.0}, + {"job_id": "leg-2", "attempt_count": 6, "spend": 0.0}, + {"job_id": "leg-3", "attempt_count": 5, "spend": 0.0}, + {"job_id": "leg-4", "attempt_count": 3, "spend": 0.0}, ] monkeypatch.setattr(proxy_server, "prisma_client", prisma) @@ -1065,7 +1079,7 @@ async def test_recorded_operator_stop_outranks_budget_arithmetic(monkeypatch: py stamp = datetime.now(timezone.utc) prisma = _shadow_prisma(legs=[_leg_record(max_turns=5, stopped_at=stamp, stopped_by="admin")]) - prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}] + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6, "spend": 0.0}] monkeypatch.setattr(proxy_server, "prisma_client", prisma) jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) @@ -1085,7 +1099,7 @@ async def test_backfilled_legacy_stop_never_reads_as_completion(monkeypatch: pyt prisma = _shadow_prisma( legs=[_leg_record(max_turns=5, stopped_at=datetime.now(timezone.utc), stopped_by="unknown")] ) - prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}] + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6, "spend": 0.0}] monkeypatch.setattr(proxy_server, "prisma_client", prisma) jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) @@ -1108,12 +1122,39 @@ def test_stopped_by_migration_backfills_every_job_that_displayed_stopped(): assert "WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc')" in sql +def test_a_start_request_still_sending_max_turns_is_rejected_not_silently_defaulted(): + """Pydantic ignores unknown fields, so without the explicit rejection a caller still + sending the retired turn budget would silently run on the default dollar budget.""" + with pytest.raises(ValidationError, match="max_budget"): + _start_request(max_turns=200) + + +def test_max_budget_migration_is_additive_and_leaves_legacy_rows_null(): + """max_budget stays NULL on pre-migration rows so they keep the turn budget they were + configured with, and shadow_cost defaults to 0 so old rows price as judge-only.""" + import litellm_proxy_extras + + sql = ( + Path(litellm_proxy_extras.__file__).parent + / "migrations" + / "20260819000000_shadow_eval_max_budget" + / "migration.sql" + ).read_text() + assert 'ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN "max_budget" DOUBLE PRECISION' in sql + assert ( + 'ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN "shadow_cost" DOUBLE PRECISION NOT NULL DEFAULT 0' + in sql + ) + assert "UPDATE" not in sql + assert "DROP" not in sql + + @pytest.mark.asyncio async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pytest.MonkeyPatch): import litellm.proxy.proxy_server as proxy_server prisma = _shadow_prisma(legs=[_leg_record(max_turns=3)]) - prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 3}] + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 3, "spend": 0.0}] monkeypatch.setattr(proxy_server, "prisma_client", prisma) with pytest.raises(HTTPException) as exhausted: @@ -1123,6 +1164,71 @@ async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pyt prisma.db.litellm_shadowevaljob.update_many.assert_not_called() +@pytest.mark.asyncio +async def test_list_reads_completed_once_every_key_spends_its_dollar_budget(monkeypatch: pytest.MonkeyPatch): + """A spend-budgeted job completes on dollars, not turns: every key's recorded shadow + plus judge spend reaching max_budget reads completed long before the turn valve, while + one key with budget left keeps the whole job running.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma( + legs=[ + _leg_record(max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0), + _leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0), + _leg_record( + id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0 + ), + ] + ) + prisma.attempt_rows = [ + {"job_id": "leg-1", "attempt_count": 40, "spend": 1.0}, + {"job_id": "leg-2", "attempt_count": 55, "spend": 1.25}, + {"job_id": "leg-3", "attempt_count": 40, "spend": 0.99}, + ] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + + by_id = {job.job_id: job for job in jobs} + assert by_id["job-1"].status == "completed" + assert by_id["job-2"].status == "running" + assert {key.api_key_id: key.spend for key in by_id["job-1"].keys} == {"key-hash": 1.0, "key-hash-2": 1.25} + assert all(key.max_budget == 1.0 for key in by_id["job-1"].keys) + + +@pytest.mark.asyncio +async def test_stop_rejects_a_job_whose_dollar_budget_is_spent(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=0.5)]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 7, "spend": 0.5}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + with pytest.raises(HTTPException) as exhausted: + await stop_shadow_eval_job("job-1", ADMIN) + assert exhausted.value.status_code == 400 + assert "completed" in exhausted.value.detail + prisma.db.litellm_shadowevaljob.update_many.assert_not_called() + + +@pytest.mark.asyncio +async def test_legacy_jobs_without_a_dollar_budget_stay_turn_gated(monkeypatch: pytest.MonkeyPatch): + """A job from before spend budgets existed carries max_budget NULL: recorded spend + can never complete it, only its own max_turns can, so migration changes nothing about + what it was configured to do.""" + import litellm.proxy.proxy_server as proxy_server + + prisma = _shadow_prisma(legs=[_leg_record(max_turns=200, max_budget=None)]) + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 40, "spend": 250.0}] + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + + jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50) + + assert jobs[0].status == "running" + assert jobs[0].keys[0].max_budget is None + assert jobs[0].keys[0].spend == 250.0 + + @pytest.mark.asyncio async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest.MonkeyPatch): import litellm.proxy.proxy_server as proxy_server @@ -1167,6 +1273,9 @@ async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_runnin assert "WHERE group_id = $1 AND stopped_by IS NULL" in stop_sql assert "ends_at > (NOW() AT TIME ZONE 'utc')" in stop_sql assert ") < k.max_turns" in stop_sql + assert "k.max_budget IS NULL" in stop_sql + assert ") < k.max_budget" in stop_sql + assert "SUM(a.judge_cost + a.shadow_cost)" in stop_sql assert (stop_group, stop_operator) == ("job-1", "admin") assert datetime.fromisoformat(stop_stamp).tzinfo is None assert prisma.db.execute_raw.await_count == 1 @@ -1357,7 +1466,7 @@ async def test_a_stop_racing_the_last_budgeted_attempt_reports_completed_not_sto import litellm.proxy.proxy_server as proxy_server prisma = _shadow_prisma(legs=[_leg_record(max_turns=2)]) - prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 2}] + prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 2, "spend": 0.0}] monkeypatch.setattr(proxy_server, "prisma_client", prisma) with pytest.raises(HTTPException) as exc: diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx index b72ebdc9c07..bceddf1eb7b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx @@ -80,7 +80,9 @@ const job = (overrides: Partial = {}): ShadowEvalJob => ({ keys: [ { api_key_id: "hashed-key-abc", - max_turns: 200, + max_turns: 10000, + max_budget: 10, + spend: 3.21, stopped_at: null, key_alias: "prod-alpha", key_name: "sk-...alpha", @@ -133,7 +135,9 @@ const keyEntry = ( overrides: Partial = {}, ): ShadowEvalJob["keys"][number] => ({ api_key_id, - max_turns: 200, + max_turns: 10000, + max_budget: 10, + spend: 0, stopped_at: null, attempt_count: null, key_alias: null, @@ -321,6 +325,21 @@ describe("ShadowEvalSection", () => { expect(screen.getByText(/ends in 3 days/)).toBeInTheDocument(); }); + it("shows recorded eval spend against the job's dollar budget", () => { + const j = job(); + mockHooks({ jobs: [j], detailsById: { "job-1": j } }); + render(); + expect(screen.getByText(/\$3\.21 of \$10\.00 eval spend/)).toBeInTheDocument(); + }); + + it("shows spend without a budget cap for a job from before spend budgets existed", () => { + const j = job({ keys: [keyEntry("hashed-key-abc", { max_budget: null, spend: 3.21 })] }); + mockHooks({ jobs: [j], detailsById: { "job-1": j } }); + render(); + expect(screen.getByText(/\$3\.21 eval spend/)).toBeInTheDocument(); + expect(screen.queryByText(/of \$/)).not.toBeInTheDocument(); + }); + it("flags rows with fewer than 30 judged turns as low sample", () => { const j = job(); mockHooks({ jobs: [j], detailsById: { "job-1": j } }); @@ -388,7 +407,7 @@ describe("ShadowEvalSection", () => { direction: "forward", shadow_percentage: 10, duration_days: 7, - max_turns: 200, + max_budget: 10, judge_model: "anthropic/claude-sonnet-5", }; expect(start.mutate).toHaveBeenCalledWith(expectedBody); @@ -425,7 +444,7 @@ describe("ShadowEvalSection", () => { baseline_model: "prod-claude", shadow_percentage: 10, duration_days: 7, - max_turns: 200, + max_budget: 10, judge_model: "anthropic/claude-sonnet-5", }; expect(start.mutate).toHaveBeenCalledWith(expectedBody); @@ -463,8 +482,8 @@ describe("ShadowEvalSection", () => { job({ judged_count: 205, keys: [ - keyEntry("hash-spent", { max_turns: 200, stopped_at: "2026-08-08T00:00:00Z" }), - keyEntry("hash-hungry", { max_turns: 500 }), + keyEntry("hash-spent", { max_budget: 2, spend: 1.5, stopped_at: "2026-08-08T00:00:00Z" }), + keyEntry("hash-hungry", { max_budget: 5, spend: 0.2 }), ], results: { by_tier: [], @@ -492,25 +511,26 @@ describe("ShadowEvalSection", () => { if (!spent || !hungry) throw new Error("expected a table row per scoped key"); expect(within(spent).getByText("stopped")).toBeInTheDocument(); - expect(within(spent).getByText("200 / 200")).toBeInTheDocument(); + expect(within(spent).getByText("$1.50 / $2.00")).toBeInTheDocument(); expect(within(spent).getByText("60.0%")).toBeInTheDocument(); expect(within(hungry).getByText("running")).toBeInTheDocument(); - expect(within(hungry).getByText("0 / 500")).toBeInTheDocument(); + expect(within(hungry).getByText("$0.2000 / $5.00")).toBeInTheDocument(); expect(within(hungry).getByText("No verdicts yet")).toBeInTheDocument(); - expect(screen.getByText(/205 of 700 turns judged/)).toBeInTheDocument(); + expect(screen.getByText(/205 turns judged/)).toBeInTheDocument(); expect(screen.getByText(/Shadowing 10% of/)).toBeInTheDocument(); expect(screen.getByText("2 keys")).toBeInTheDocument(); }); it("reads a key that spent its budget as completed even before the sweep stamps it", () => { + const legacyTurnBudgetLeg = { max_budget: null, spend: 0.5, max_turns: 500, attempt_count: 3 }; mockHooks({ jobs: [ job({ keys: [ - keyEntry("hash-spent", { max_turns: 200, attempt_count: 200 }), - keyEntry("hash-hungry", { max_turns: 500, attempt_count: 3 }), + keyEntry("hash-spent", { max_budget: 2, spend: 2, attempt_count: 40 }), + keyEntry("hash-hungry", legacyTurnBudgetLeg), ], }), ], @@ -521,9 +541,9 @@ describe("ShadowEvalSection", () => { const hungry = screen.getByText("hash-hungr…").closest("tr"); if (!spent || !hungry) throw new Error("expected a table row per scoped key"); expect(within(spent).getByText("completed")).toBeInTheDocument(); - expect(within(spent).getByText("200 / 200")).toBeInTheDocument(); + expect(within(spent).getByText("$2.00 / $2.00")).toBeInTheDocument(); expect(within(hungry).getByText("running")).toBeInTheDocument(); - expect(within(hungry).getByText("3 / 500")).toBeInTheDocument(); + expect(within(hungry).getByText("3 / 500 turns")).toBeInTheDocument(); }); it("shows the per-key table while a multi-key job is still collecting, before any verdicts exist", () => { @@ -533,8 +553,8 @@ describe("ShadowEvalSection", () => { judged_count: 0, results: null, keys: [ - keyEntry("hash-spent", { max_turns: 2, attempt_count: 2 }), - keyEntry("hash-hungry", { max_turns: 500, attempt_count: 1 }), + keyEntry("hash-spent", { max_budget: 0.5, spend: 0.5, attempt_count: 2 }), + keyEntry("hash-hungry", { max_budget: 5, spend: 0.01, attempt_count: 1 }), ], }), ], @@ -544,7 +564,7 @@ describe("ShadowEvalSection", () => { const spent = screen.getByText("hash-spent…").closest("tr"); if (!spent) throw new Error("expected a per-key row before verdicts exist"); expect(within(spent).getByText("completed")).toBeInTheDocument(); - expect(within(spent).getByText("2 / 2")).toBeInTheDocument(); + expect(within(spent).getByText("$0.5000 / $0.5000")).toBeInTheDocument(); expect(screen.getByText("Budget used")).toBeInTheDocument(); expect(screen.queryByText("Judged turns")).not.toBeInTheDocument(); expect(screen.getByText(/Collecting verdicts/)).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx index bc5044feaa6..44054e3b7c4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx @@ -57,9 +57,19 @@ export const shadowedKeyLabel = (key: ShadowEvalJobKey): string => const shadowedKeysLabel = (job: ShadowEvalJob): string => job.keys.length === 1 ? shadowedKeyLabel(job.keys[0]) : `${job.keys.length} keys`; -const totalBudget = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + key.max_turns, 0); +const totalBudget = (job: ShadowEvalJob): number | null => + job.keys.reduce( + (sum, key) => (sum === null || key.max_budget == null ? null : sum + key.max_budget), + 0, + ); -const keySpent = (key: ShadowEvalJobKey): boolean => key.attempt_count != null && key.attempt_count >= key.max_turns; +const totalSpend = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + (key.spend ?? 0), 0); + +const keySpent = (key: ShadowEvalJobKey): boolean => { + const spendBudgetReached = key.max_budget != null && key.spend != null && key.spend >= key.max_budget; + const turnValveReached = key.attempt_count != null && key.attempt_count >= key.max_turns; + return spendBudgetReached || turnValveReached; +}; const keyStatus = (job: ShadowEvalJob, key: ShadowEvalJobKey): string => { if (job.status === "completed" || (key.stopped_at == null && keySpent(key))) return "completed"; @@ -207,7 +217,9 @@ const KeyTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => { - {(key.attempt_count ?? slice?.turn_count ?? 0).toLocaleString()} / {key.max_turns.toLocaleString()} + {key.max_budget != null + ? `${usd(key.spend ?? 0)} / ${usd(key.max_budget)}` + : `${(key.attempt_count ?? slice?.turn_count ?? 0).toLocaleString()} / ${key.max_turns.toLocaleString()} turns`} {slice ? ( <> @@ -300,8 +312,9 @@ const JobResults: React.FC<{

{jobHeadline(job)}

- {(job.judged_count ?? 0).toLocaleString()} of {totalBudget(job).toLocaleString()} turns judged ·{" "} - {(job.error_count ?? 0).toLocaleString()} errored · {usd(job.judge_spend ?? 0)} judge spend + {(job.judged_count ?? 0).toLocaleString()} turns judged · {(job.error_count ?? 0).toLocaleString()}{" "} + errored · {usd(totalSpend(job))} + {totalBudget(job) !== null ? ` of ${usd(totalBudget(job) ?? 0)}` : ""} eval spend {active && remaining ? ` · ${remaining}` : ""}

@@ -375,9 +388,9 @@ const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[ const START_FORM_DESCRIPTION: Record = { forward: - "Duplicates a sampled slice of the selected keys' traffic through the auto-router and has an LLM judge compare both answers blind. Each key gets its own turn budget. The router's answers are never served to users; judge calls bill to the shadowed key.", + "Duplicates a sampled slice of the selected keys' traffic through the auto-router and has an LLM judge compare both answers blind. Each key gets its own spend budget. The router's answers are never served to users; judge calls bill to the shadowed key.", reverse: - "Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each key gets its own turn budget. The baseline's answers are never served to users; judge calls bill to the shadowed key.", + "Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each key gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the shadowed key.", }; const DURATION_OPTIONS = [ @@ -445,7 +458,7 @@ const StartForm: React.FC = () => { const [percentage, setPercentage] = useState("10"); const [durationDays, setDurationDays] = useState("7"); const [judgeModel, setJudgeModel] = useState(""); - const [maxTurns, setMaxTurns] = useState("200"); + const [maxBudget, setMaxBudget] = useState("10"); const { data: autoRouters } = useAutoRouters(); const judgeModelOptions = useJudgeModelOptions(); const baselineModelOptions = useBaselineModelOptions(); @@ -460,11 +473,11 @@ const StartForm: React.FC = () => { const parsedPct = Number.parseFloat(percentage); const percentageValid = parsedPct >= 0.1 && parsedPct <= 100; - const parsedMaxTurns = Number.parseInt(maxTurns, 10); - const maxTurnsValid = parsedMaxTurns >= 1 && parsedMaxTurns <= 2000; + const parsedMaxBudget = Number.parseFloat(maxBudget); + const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000; const baselinePicked = direction === "forward" || baselineModel !== ""; const filled = apiKeyIds.length > 0 && [routerName, judgeModel].every((field) => field !== "") && baselinePicked; - const boundsValid = percentageValid && maxTurnsValid; + const boundsValid = percentageValid && maxBudgetValid; const valid = Boolean(accessToken) && filled && boundsValid; const handleStart = () => { const startBody = { @@ -474,7 +487,7 @@ const StartForm: React.FC = () => { ...(direction === "reverse" ? { baseline_model: baselineModel } : {}), shadow_percentage: parsedPct, duration_days: Number.parseInt(durationDays, 10), - max_turns: parsedMaxTurns, + max_budget: parsedMaxBudget, judge_model: judgeModel, }; start.mutate(startBody); @@ -551,20 +564,22 @@ const StartForm: React.FC = () => { - +
+ $ setMaxTurns(e.target.value)} + value={maxBudget} + onChange={(e) => setMaxBudget(e.target.value)} /> - turns judged, max + max shadow + judge spend, per key
- {maxTurns.trim() !== "" && !maxTurnsValid && ( -

Enter a value from 1 to 2000

+ {maxBudget.trim() !== "" && !maxBudgetValid && ( +

Enter a value from 0.01 to 10000

)} {direction === "reverse" && ( @@ -620,7 +635,7 @@ const PreviousJob: React.FC<{ job: ShadowEvalJob }> = ({ job }) => {

{jobHeadline(shown)}

{shown.judged_count != null && - `${shown.judged_count.toLocaleString()} judged · ${(shown.error_count ?? 0).toLocaleString()} errored · ${usd(shown.judge_spend ?? 0)} judge spend · `} + `${shown.judged_count.toLocaleString()} judged · ${(shown.error_count ?? 0).toLocaleString()} errored · ${usd(totalSpend(shown))} eval spend · `} {new Date(shown.created_at).toLocaleDateString()}

diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 790ebcdd227..847fe65a5cc 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -849,11 +849,12 @@ export interface paths { * serve and duplicates them against baseline_model. A key can hold one active job per * direction, so both questions can run at once. * - * Shadow responses are never served to users. Each key samples until it has judged - * max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one - * key running out of budget does not end sampling for the others; sampling changes - * propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed - * key but are excluded from request counts and auto-router adoption metrics. + * Shadow responses are never served to users. Each key samples until its recorded eval + * spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's + * window ends, or the job is stopped, so one key running out of budget does not end + * sampling for the others; sampling changes propagate to pods within about 10 seconds. + * Shadow and judge calls bill to the shadowed key but are excluded from request counts + * and auto-router adoption metrics. */ post: operations["start_shadow_eval_auto_router_shadow_eval_start_post"]; delete?: never; @@ -33305,11 +33306,21 @@ export interface components { * @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias */ key_name?: string | null; + /** + * Max Budget + * @description This key's own USD budget for the eval's shadow and judge spend, independent of its siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds + */ + max_budget?: number | null; /** * Max Turns - * @description This key's own sample budget, independent of its siblings' + * @description This key's sample-count ceiling: the whole budget for jobs created before max_budget existed, and the error-loop safety valve otherwise */ max_turns: number; + /** + * Spend + * @description This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets against max_budget; populated on list and detail responses and frozen at stopped_at exactly like attempt_count + */ + spend?: number | null; /** * Stopped At * @description When this key's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset @@ -33626,7 +33637,7 @@ export interface components { StartShadowEvalRequest: { /** * Api Key Ids - * @description The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these keys' traffic; requests made with any other key are not sampled. Each key carries its own max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 keys per job, which also bounds every read the job's endpoints make. + * @description The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these keys' traffic; requests made with any other key are not sampled. Each key carries its own max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 keys per job, which also bounds every read the job's endpoints make. */ api_key_ids: string[]; /** @@ -33654,11 +33665,11 @@ export interface components { */ judge_model: string; /** - * Max Turns - * @description Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, so a job over N keys judges at most N times max_turns turns. This is also the spend bound; expected judge cost is roughly that turn ceiling times one judge call - * @default 200 + * Max Budget + * @description Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window + * @default 10 */ - max_turns: number; + max_budget: number; /** * Router Name * @description The auto-router under evaluation, in either direction