scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 48

feat(shadow_eval)!: gate the per-key budget on dollar spend instead of turns (#37555)

Browse source
This commit is contained in:
tin-berri 2026-08-20 14:55:21 -07:00 • committed by GitHub
parent 60e03bedcf
commit 2dcd453860
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 671 additions and 138 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_shadow_eval_max_budget/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN "max_budget" DOUBLE PRECISION;
-- AlterTable
ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN "shadow_cost" DOUBLE PRECISION NOT NULL DEFAULT 0;

4
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1502,7 +1502,8 @@ model LiteLLM_ShadowEvalJob {
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
shadow_percentage Float
max_turns Int // this key's sample budget: judge at most this many turns
max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
created_at DateTime @default(now())
created_by String?
ends_at DateTime
@ -1525,6 +1526,7 @@ model LiteLLM_ShadowEvalAttempt {
shadow_model String?
confidence Float?
judge_cost Float @default(0)
shadow_cost Float @default(0)
error String?
created_at DateTime @default(now())

133
litellm/integrations/shadow_eval_logger.py
View file

@ -10,7 +10,7 @@ import asyncio
import hashlib
import random
import traceback
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from itertools import groupby
@ -42,8 +42,9 @@ if TYPE_CHECKING:
from litellm.router import Router
from litellm.types.utils import StandardLoggingPayload
# A job starting, stopping, or hitting its turn budget propagates to sampling within one
# TTL; the turn budget can overshoot by at most one TTL of in-flight samples per pod.
# A job starting, stopping, or hitting a budget propagates to sampling within one TTL;
# the spend gate re-checks the cross-pod counter at pipeline entry, so it overshoots
# only by the samples already in flight when the cap is crossed.
_JOBS_CACHE_TTL_SECONDS: Final = 10
# Concurrent shadow+judge pipelines per pod: a traffic spike turns into skipped samples
@ -340,13 +341,24 @@ def _failure_detail(e: BaseException) -> str:
return f"{type(e).__name__}{location}: {e}"
def _judge_call_cost(response: object) -> float:
"""Price a judge call, treating an unmapped judge model as free rather than fatal."""
def _call_cost(response: object) -> float:
"""Price one eval-arm call with the figure the spend pipeline bills: the router client
stamps _hidden_params.response_cost from the deployment's own pricing, which the public
price map lookup below cannot see (it reads 0 for deployment-priced models)."""
getter: Final = getattr(getattr(response, "_hidden_params", None), "get", None)
stamped: Final = getter("response_cost") if callable(getter) else None
if isinstance(stamped, (int, float)):
return float(stamped)
return _price_map_cost(response)
def _price_map_cost(response: object) -> float:
"""Public price map fallback, treating an unmapped model as free rather than fatal."""
import litellm
try:
return litellm.completion_cost(completion_response=response) or 0.0
except Exception: # noqa: BLE001 # unmapped judge model: the verdict still counts, cost stays 0
except Exception: # noqa: BLE001 # unmapped model: the attempt still counts, cost stays 0
return 0.0
@ -374,6 +386,32 @@ def _judge_user_prompt(conversation: str, response_a: str, response_b: str) -> s
)
def _job_spend_counter_key(job_id: str) -> str:
return f"spend:shadow_eval:{job_id}"
async def _job_spend_from_counter(counter_key: str, fallback_spend: float, max_budget: float) -> float:
"""The leg's spend through the cross-pod counter the key budget gates read. The owner
degrades internally to the fill-time DB floor and raises only under fail-closed
enforcement, which the caller honors by skipping the sample."""
from litellm.proxy.proxy_server import get_current_spend
return await get_current_spend(counter_key=counter_key, fallback_spend=fallback_spend, max_budget=max_budget)
async def _add_job_spend_to_counter(counter_key: str, cost: float) -> None:
"""Advance the counter the moment a cost is known, so even a lost row closes the gate.
Known failure mode: a Redis outage freezes the counter (the owner invalidates it), the
gate degrades to the fill floor, and overshoot grows to in-flight plus one TTL of
samples, the same degradation the key budget counters accept."""
try:
from litellm.proxy.proxy_server import increment_spend_counter
await increment_spend_counter(counter_key=counter_key, increment=cost)
except Exception as e: # noqa: BLE001 # attempt recording must proceed; the row stays truth and the fill floor gates
verbose_logger.warning("shadow_eval: spend counter increment failed for %s: %s", counter_key, e)
async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
"""Whether the shadowed key or its team is over budget, decided by the same owners
the request path uses, so counter keys and thresholds can never drift from auth's.
@ -438,8 +476,8 @@ def _request_was_routed_by(request_metadata: Mapping[str, object], router_name:
@dataclass(frozen=True, slots=True)
class _CallFailure:
"""A shadow or judge call that produced no usable response. cost carries any judge
spend the failed attempt still billed, so job-level judge_spend never undercounts."""
"""A shadow or judge call that produced no usable response. cost carries any spend
the failed call still billed, so job-level spend figures never undercount."""
error: str
cost: float = 0.0
@ -452,6 +490,7 @@ class _ShadowResponse:
text: str
model: str
tier: str | None
cost: float
@dataclass(frozen=True, slots=True)
@ -478,8 +517,10 @@ class ActiveShadowEvalJob(BaseModel):
shadow_percentage: float
judge_model: str
max_turns: int
max_budget: float | None = None
ends_at: datetime
attempts: int = 0
spend: float = 0.0
@field_validator("ends_at")
@classmethod
@ -500,7 +541,7 @@ class ActiveShadowEvalJob(BaseModel):
return self.baseline_model or self.router_name
def _as_active_job(record: object, attempts: int) -> ActiveShadowEvalJob | None:
def _as_active_job(record: object, attempts: int, spend: float) -> ActiveShadowEvalJob | None:
"""The sampling path's view of one job row, or None for a row it cannot sample: an
unknown direction, or a reverse job with no baseline model to duplicate against.
Failing closed here is what keeps the dispatch path total."""
@ -509,7 +550,7 @@ def _as_active_job(record: object, attempts: int) -> ActiveShadowEvalJob | None:
except ValidationError as e:
verbose_logger.debug("shadow_eval: skipping unsamplable job row: %s", e)
return None
return job.model_copy(update={"attempts": attempts})
return job.model_copy(update={"attempts": attempts, "spend": spend}) # mutable-ok: pydantic update payload
_jobs_cache: Final = InMemoryCache(max_size_in_memory=4, default_ttl=_JOBS_CACHE_TTL_SECONDS)
@ -524,12 +565,17 @@ class ShadowEvalLogger(CustomLogger):
router_provider: Callable[[], "Router | None"] | None = None,
prisma_provider: Callable[[], "PrismaClient | None"] | None = None,
jobs_cache: InMemoryCache | None = None,
job_spend_reader: Callable[[str, float, float], Awaitable[float]] | None = None,
job_spend_writer: Callable[[str, float], Awaitable[None]] | None = None,
) -> None:
"""Providers are callables so the proxy's lazily-initialized globals are resolved
at call time, not at logger construction."""
at call time, not at logger construction. The spend reader and writer wrap the
proxy's cross-pod spend counter; tests inject a plain in-memory pair."""
self._router_provider = router_provider or default_router_provider
self._prisma_provider = prisma_provider or _default_prisma_provider
self._jobs_cache = jobs_cache or _jobs_cache
self._read_job_spend = job_spend_reader or _job_spend_from_counter
self._write_job_spend = job_spend_writer or _add_job_spend_to_counter
self._inflight_shadow_tasks: int = 0
# Starts per job since the last cache fill, never decremented within a
# generation; the refill absorbs written rows and resets.
@ -556,18 +602,26 @@ class ShadowEvalLogger(CustomLogger):
await prisma.db.litellm_shadowevalattempt.group_by(
by=["job_id"],
count=True,
sum={"judge_cost": True, "shadow_cost": True}, # mutable-ok: Prisma aggregate spec
where={"job_id": {"in": [str(record.id) for record in records]}}, # mutable-ok: Prisma filter
)
if records
else ()
)
attempt_counts: Final = {str(row["job_id"]): int(row["_count"]["_all"]) for row in grouped or []}
attempt_stats: Final = { # mutable-ok: frozen snapshot of the grouped read
str(row["job_id"]): (
int(row["_count"]["_all"]),
float((row["_sum"] or {}).get("judge_cost") or 0.0)
+ float((row["_sum"] or {}).get("shadow_cost") or 0.0),
)
for row in grouped or []
}
by_key: Final = tuple(
sorted(
(
(str(record.api_key_id), job)
for record in records or []
if (job := _as_active_job(record, attempt_counts.get(str(record.id), 0))) is not None
if (job := _as_active_job(record, *attempt_stats.get(str(record.id), (0, 0.0)))) is not None
),
key=itemgetter(0),
)
@ -624,6 +678,7 @@ class ShadowEvalLogger(CustomLogger):
for job in (await self._active_jobs()).get(str(api_key_hash), ())
if datetime.now(timezone.utc) < job.ends_at
and job.attempts + self._job_starts.get(job.id, 0) < job.max_turns
and (job.max_budget is None or job.spend < job.max_budget)
and _sample_hits(request_id, job.id, job.shadow_percentage)
and _request_was_routed_by(request_metadata, job.router_name) == (job.direction == "reverse")
)
@ -684,12 +739,28 @@ class ShadowEvalLogger(CustomLogger):
return
if await _key_or_team_is_over_budget(parent_metadata):
return
if job.max_budget is not None:
try:
spend: Final = await self._read_job_spend(_job_spend_counter_key(job.id), job.spend, job.max_budget)
except Exception as e: # noqa: BLE001 # unverifiable budget: skip the sample rather than spend on it
verbose_logger.warning("shadow_eval: budget unverifiable for %s, sample skipped: %s", job.id, e)
return
if spend >= job.max_budget:
return
shadow: Final = await self._call_router_shadow(job.shadow_target, messages, shadow_params, parent_metadata)
if isinstance(shadow, _CallFailure):
await self._record_attempt(prisma, job, request_id, control_tier, outcome="error", error=shadow.error)
return
except Exception as e: # noqa: BLE001 # detached task: nothing billed yet, record and never raise
verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
await self._record_attempt(
prisma, job, request_id, control_tier, outcome="error", error=f"pipeline error: {e}"
)
return
if isinstance(shadow, _CallFailure):
await self._record_attempt(
prisma, job, request_id, control_tier, outcome="error", error=shadow.error, shadow_cost=shadow.cost
)
return
# From here the shadow call has billed, so every exit records its cost.
try:
verdict: Final = await self._call_judge(
judge_model=job.judge_model,
messages=messages,
@ -707,6 +778,7 @@ class ShadowEvalLogger(CustomLogger):
error=verdict.error,
shadow=shadow,
judge_cost=verdict.cost,
shadow_cost=shadow.cost,
)
return
await self._record_attempt(
@ -719,15 +791,23 @@ class ShadowEvalLogger(CustomLogger):
real_model=real_model,
confidence=verdict.confidence,
judge_cost=verdict.cost,
shadow_cost=shadow.cost,
)
except Exception as e: # noqa: BLE001 # detached task: record what happened, never raise
except Exception as e: # noqa: BLE001 # detached task: the shadow call billed, record its cost, never raise
verbose_logger.debug("shadow_eval: pipeline failed for %s: %s", request_id, e)
await self._record_attempt(
prisma, job, request_id, control_tier, outcome="error", error=f"pipeline error: {e}"
prisma,
job,
request_id,
control_tier,
outcome="error",
error=f"pipeline error: {e}",
shadow=shadow,
shadow_cost=shadow.cost,
)
@staticmethod
async def _record_attempt(
self,
prisma: "PrismaClient | None",
job: ActiveShadowEvalJob,
request_id: str,
@ -738,8 +818,11 @@ class ShadowEvalLogger(CustomLogger):
real_model: str = "",
confidence: float | None = None,
judge_cost: float = 0.0,
shadow_cost: float = 0.0,
error: str | None = None,
) -> None:
if judge_cost + shadow_cost > 0:
await self._write_job_spend(_job_spend_counter_key(job.id), judge_cost + shadow_cost)
if prisma is None:
return
try:
@ -753,6 +836,7 @@ class ShadowEvalLogger(CustomLogger):
"shadow_model": shadow.model if shadow else None,
"confidence": confidence,
"judge_cost": judge_cost,
"shadow_cost": shadow_cost,
"error": error[:_MAX_ERROR_CHARS] if error else None,
}
)
@ -792,11 +876,12 @@ class ShadowEvalLogger(CustomLogger):
return _CallFailure(f"shadow router call failed: {_failure_detail(e)}")
text: Final = _chat_final_text(response)
if not text:
return _CallFailure("shadow router returned an empty response")
return _CallFailure("shadow router returned an empty response", cost=_call_cost(response))
return _ShadowResponse(
text=text,
model=str(getattr(response, "model", None) or _routing_decision(shadow_metadata).get("routed_model") or ""),
tier=_routed_tier(shadow_metadata),
cost=_call_cost(response),
)
async def _call_judge(
@ -843,11 +928,11 @@ class ShadowEvalLogger(CustomLogger):
verdict: Final = PairwiseVerdict.model_validate(parse_json_verdict(raw))
except Exception as e: # noqa: BLE001 # malformed verdicts become error rows
verbose_logger.debug("shadow_eval: unparseable judge verdict: %s", e)
return _CallFailure(f"unparseable judge verdict: {e}", cost=_judge_call_cost(response))
return _CallFailure(f"unparseable judge verdict: {e}", cost=_call_cost(response))
return _JudgeVerdict(
preference=_unmask_preference(verdict.preference, real_is_a),
confidence=max(0.0, min(1.0, verdict.confidence)),
cost=_judge_call_cost(response),
cost=_call_cost(response),
)

58
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -37,6 +37,7 @@ from litellm.repositories.base_repository import SupportsModelDump
from litellm.repositories.team_repository import TeamRepository
from litellm.router_strategy.complexity_router import ComplexityRouter
from litellm.types.management_endpoints.auto_router_endpoints import (
SHADOW_EVAL_TURN_VALVE,
AutoRouterBenchmarkGroup,
AutoRouterBenchmarksResponse,
AutoRouterBenchmarkTotals,
@ -662,12 +663,19 @@ _ATTEMPT_AGG_BY_TIER_SQL: Final = "SELECT COALESCE(tier, 'UNCLASSIFIED') AS grp,
_ATTEMPT_AGG_BY_MODEL_SQL: Final = "SELECT COALESCE(real_model, 'unknown') AS grp," + _ATTEMPT_AGG_SELECT
_ATTEMPT_AGG_BY_LEG_SQL: Final = "SELECT job_id AS grp," + _ATTEMPT_AGG_SELECT
# These guards derive spend from attempt rows, the cross-pod authority; the sampler also
# reads the live counter, so admission can stop before a row-based guard would fire (safe
# direction, and mid-deploy rows from old pods price as judge-only until the deploy ends).
_SWEEP_FINISHED_JOBS_SQL: Final = """
UPDATE "LiteLLM_ShadowEvalJob" j SET stopped_at = (NOW() AT TIME ZONE 'utc')
WHERE j.api_key_id = ANY($1::text[]) AND j.stopped_at IS NULL
AND (
j.ends_at <= (NOW() AT TIME ZONE 'utc')
OR (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_turns
OR (
j.max_budget IS NOT NULL
AND (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = j.id) >= j.max_budget
)
)
"""
@ -681,7 +689,7 @@ WHERE job_id = ANY($1::text[])
"""
_ATTEMPT_COUNTS_SQL: Final = """
SELECT a.job_id, COUNT(*)::int AS attempt_count
SELECT a.job_id, COUNT(*)::int AS attempt_count, COALESCE(SUM(a.judge_cost + a.shadow_cost), 0)::float AS spend
FROM "LiteLLM_ShadowEvalAttempt" a
JOIN "LiteLLM_ShadowEvalJob" j ON j.id = a.job_id
WHERE a.job_id = ANY($1::text[]) AND (j.stopped_at IS NULL OR a.created_at <= j.stopped_at)
@ -697,6 +705,10 @@ WHERE group_id = $1 AND stopped_by IS NULL
SELECT 1 FROM "LiteLLM_ShadowEvalJob" k
WHERE k.group_id = $1 AND k.stopped_at IS NULL
AND (SELECT COUNT(*) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_turns
AND (
k.max_budget IS NULL
OR (SELECT COALESCE(SUM(a.judge_cost + a.shadow_cost), 0) FROM "LiteLLM_ShadowEvalAttempt" a WHERE a.job_id = k.id) < k.max_budget
)
)
"""
@ -704,6 +716,7 @@ WHERE group_id = $1 AND stopped_by IS NULL
class _AttemptCountRow(BaseModel):
job_id: str
attempt_count: int
spend: float
_ATTEMPT_COUNT_ROWS: Final = TypeAdapter(list[_AttemptCountRow])
@ -770,6 +783,7 @@ class _LegRow(BaseModel):
judge_model: str
shadow_percentage: float
max_turns: int
max_budget: float | None = None
created_at: datetime
ends_at: datetime
stopped_at: datetime | None = None
@ -789,22 +803,25 @@ class _LegRow(BaseModel):
_LEG_ROWS: Final = TypeAdapter(list[_LegRow])
async def _leg_attempt_counts(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> Mapping[str, int]:
"""Each leg's attempt count by leg id, judged and errored alike, in one grouped read.
It is the same count the sampler budgets against max_turns, so the derived status
flips to completed exactly when sampling actually ends. A stamped leg's count freezes
at its stopped_at: in-flight attempts that land after the stamp are excluded, so they
can never reclassify a leg that was stopped under budget as budget-spent."""
async def _leg_attempt_counts(prisma_client: "PrismaClient", legs: Sequence[_LegRow]) -> Mapping[str, _AttemptCountRow]:
"""Each leg's attempt count and recorded spend by leg id, judged and errored alike, in
one grouped read. They are the same figures the sampler budgets against max_turns and
max_budget, so the derived status flips to completed exactly when sampling actually
ends. A stamped leg's figures freeze at its stopped_at: in-flight attempts that land
after the stamp are excluded, so they can never reclassify a leg that was stopped
under budget as budget-spent."""
if not legs:
return MappingProxyType({})
rows: Final = _ATTEMPT_COUNT_ROWS.validate_python(
await _query_raw(prisma_client, _ATTEMPT_COUNTS_SQL, [leg.id for leg in legs]) # mutable-ok: query param
or ()
)
return MappingProxyType({row.job_id: row.attempt_count for row in rows})
return MappingProxyType({row.job_id: row for row in rows})
def _group_response(group_id: str, legs: Sequence[_LegRow], attempt_counts: Mapping[str, int]) -> ShadowEvalJobResponse:
def _group_response(
group_id: str, legs: Sequence[_LegRow], attempt_counts: Mapping[str, _AttemptCountRow]
) -> ShadowEvalJobResponse:
"""The one constructor of a job response: the caller names the group and passes that
group's legs. Config is read off the first leg because every leg carries the same copy,
written by one create_many. No caller may serialize a raw row (that would leak a leg id
@ -816,8 +833,10 @@ def _group_response(group_id: str, legs: Sequence[_LegRow], attempt_counts: Mapp
ShadowEvalJobKeyResponse(
api_key_id=leg.api_key_id,
max_turns=leg.max_turns,
max_budget=leg.max_budget,
stopped_at=leg.stopped_at,
attempt_count=attempt_counts.get(leg.id, 0),
attempt_count=stats.attempt_count if (stats := attempt_counts.get(leg.id)) else 0,
spend=round(stats.spend, 6) if stats else 0.0,
)
for leg in sorted(legs, key=lambda leg: leg.api_key_id)
),
@ -923,11 +942,12 @@ async def start_shadow_eval(
serve and duplicates them against baseline_model. A key can hold one active job per
direction, so both questions can run at once.
Shadow responses are never served to users. Each key samples until it has judged
max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one
key running out of budget does not end sampling for the others; sampling changes
propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed
key but are excluded from request counts and auto-router adoption metrics.
Shadow responses are never served to users. Each key samples until its recorded eval
spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's
window ends, or the job is stopped, so one key running out of budget does not end
sampling for the others; sampling changes propagate to pods within about 10 seconds.
Shadow and judge calls bill to the shadowed key but are excluded from request counts
and auto-router adoption metrics.
"""
from litellm.proxy.proxy_server import llm_router, prisma_client
@ -952,7 +972,7 @@ async def start_shadow_eval(
),
)
# A job whose window passed or whose turn budget ran out stopped sampling on its own,
# A job whose window passed or whose budget ran out stopped sampling on its own,
# but its legs still hold their slots in the per-key, per-direction partial unique index
# until stamped; free them so a new eval can start. Sweeping both directions is deliberate.
requested: Final = list(data.api_key_ids) # mutable-ok: query param
@ -983,7 +1003,8 @@ async def start_shadow_eval(
"baseline_model": data.baseline_model,
"judge_model": data.judge_model,
"shadow_percentage": data.shadow_percentage,
"max_turns": data.max_turns,
"max_turns": SHADOW_EVAL_TURN_VALVE,
"max_budget": data.max_budget,
"created_by": user_api_key_dict.user_id,
"created_at": now,
"ends_at": ends_at,
@ -1007,7 +1028,8 @@ async def start_shadow_eval(
keys=tuple(
ShadowEvalJobKeyResponse(
api_key_id=api_key_id,
max_turns=data.max_turns,
max_turns=SHADOW_EVAL_TURN_VALVE,
max_budget=data.max_budget,
key_alias=labels[api_key_id].key_alias,
key_name=labels[api_key_id].key_name,
)

7
litellm/proxy/proxy_server.py
View file

@ -2994,6 +2994,13 @@ async def _is_spend_counter_cache_warm(counter_key: str) -> bool:
return spend_counter_cache.in_memory_cache.get_cache(key=counter_key) is not None
async def increment_spend_counter(counter_key: str, increment: float):
"""Public raw-counter increment for budget domains outside the entity scopes (e.g.
shadow eval's per-leg spend), sharing the primitive the entity counters use so
invalidation and read semantics can never drift."""
return await _increment_spend_counter_cache(counter_key=counter_key, increment=increment)
async def _increment_spend_counter_cache(counter_key: str, increment: float):
if spend_counter_cache.redis_cache is not None:
try:

4
litellm/proxy/schema.prisma
View file

@ -1502,7 +1502,8 @@ model LiteLLM_ShadowEvalJob {
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
shadow_percentage Float
max_turns Int // this key's sample budget: judge at most this many turns
max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
created_at DateTime @default(now())
created_by String?
ends_at DateTime
@ -1525,6 +1526,7 @@ model LiteLLM_ShadowEvalAttempt {
shadow_model String?
confidence Float?
judge_cost Float @default(0)
shadow_cost Float @default(0)
error String?
created_at DateTime @default(now())

55
litellm/types/management_endpoints/auto_router_endpoints.py
View file

@ -169,6 +169,10 @@ ShadowEvalDirection: TypeAlias = Literal["forward", "reverse"]
DEFAULT_SHADOW_EVAL_JUDGE_MODEL: Final[str] = "anthropic/claude-sonnet-5"
# Sample-count ceiling written on every new job: a zero-cost error loop (a shadow arm that
# fails before billing) never consumes spend budget, so it must terminate on count instead.
SHADOW_EVAL_TURN_VALVE: Final[int] = 10_000
class StartShadowEvalRequest(BaseModel):
"""Start duplicating one or more keys' traffic for blind comparison against an auto-router."""
@ -179,7 +183,7 @@ class StartShadowEvalRequest(BaseModel):
description=(
"The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these "
"keys' traffic; requests made with any other key are not sampled. Each key carries its own "
"max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 "
"max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 "
"keys per job, which also bounds every read the job's endpoints make."
),
)
@ -219,17 +223,27 @@ class StartShadowEvalRequest(BaseModel):
le=30,
description="How many days the job samples traffic before completing on its own",
)
max_turns: int = Field(
default=200,
ge=1,
le=2000,
max_budget: float = Field(
default=10.0,
ge=0.01,
le=10_000,
description=(
"Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, "
"so a job over N keys judges at most N times max_turns turns. This is also the spend bound; "
"expected judge cost is roughly that turn ceiling times one judge call"
"Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with "
"the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval "
"spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight "
"samples can overshoot the cap by one sampling cache window"
),
)
@model_validator(mode="before")
@classmethod
def _reject_the_retired_turn_budget(cls, values: object) -> object:
"""Pydantic ignores unknown fields, so a caller still sending max_turns would
silently run on the default dollar budget instead of the bound they asked for."""
if isinstance(values, Mapping) and "max_turns" in values:
raise ValueError("max_turns was replaced by max_budget, the per-key USD cap on the eval's own spend")
return values
@field_validator("shadow_percentage")
@classmethod
def _round_percentage(cls, value: float) -> float:
@ -296,7 +310,19 @@ class ShadowEvalJobKeyResponse(BaseModel):
"""One key a job shadows, with its own budget and stop state."""
api_key_id: str = Field(description="The hashed virtual key whose traffic this entry scopes")
max_turns: int = Field(description="This key's own sample budget, independent of its siblings'")
max_turns: int = Field(
description=(
"This key's sample-count ceiling: the whole budget for jobs created before max_budget "
"existed, and the error-loop safety valve otherwise"
)
)
max_budget: float | None = Field(
default=None,
description=(
"This key's own USD budget for the eval's shadow and judge spend, independent of its "
"siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds"
),
)
stopped_at: datetime | None = Field(
default=None,
description=(
@ -313,10 +339,19 @@ class ShadowEvalJobKeyResponse(BaseModel):
"once the key is stamped, so in-flight attempts landing after a stop never reclassify it"
),
)
spend: float | None = Field(
default=None,
description=(
"This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets "
"against max_budget; populated on list and detail responses and frozen at stopped_at "
"exactly like attempt_count"
),
)
@property
def budget_spent(self) -> bool:
return self.attempt_count is not None and self.attempt_count >= self.max_turns
over_spend: Final = self.max_budget is not None and self.spend is not None and self.spend >= self.max_budget
return over_spend or (self.attempt_count is not None and self.attempt_count >= self.max_turns)
key_alias: str | None = Field(
default=None,

4
schema.prisma
View file

@ -1502,7 +1502,8 @@ model LiteLLM_ShadowEvalJob {
baseline_model String? // reverse only: the fixed model the router is judged against
judge_model String
shadow_percentage Float
max_turns Int // this key's sample budget: judge at most this many turns
max_turns Int // sample-count ceiling: the whole budget on pre-max_budget jobs, the error-loop valve otherwise
max_budget Float? // per-key USD cap on the eval's own shadow + judge spend; null on jobs from before spend budgets
created_at DateTime @default(now())
created_by String?
ends_at DateTime
@ -1525,6 +1526,7 @@ model LiteLLM_ShadowEvalAttempt {
shadow_model String?
confidence Float?
judge_cost Float @default(0)
shadow_cost Float @default(0)
error String?
created_at DateTime @default(now())

258
tests/test_litellm/integrations/test_shadow_eval_logger.py
View file

@ -40,11 +40,19 @@ def _job(**overrides) -> ActiveShadowEvalJob:
return ActiveShadowEvalJob(**{**defaults, **overrides})
def _prisma(jobs=(), attempt_counts=()) -> MagicMock:
def _prisma(jobs=(), attempt_counts=(), attempt_costs=()) -> MagicMock:
costs = {job_id: {"judge_cost": judge, "shadow_cost": shadow} for job_id, judge, shadow in attempt_costs}
prisma = MagicMock()
prisma.db.litellm_shadowevaljob.find_many = AsyncMock(return_value=list(jobs))
prisma.db.litellm_shadowevalattempt.group_by = AsyncMock(
return_value=[{"job_id": job_id, "_count": {"_all": count}} for job_id, count in attempt_counts]
return_value=[
{
"job_id": job_id,
"_count": {"_all": count},
"_sum": costs.get(job_id, {"judge_cost": 0.0, "shadow_cost": 0.0}),
}
for job_id, count in attempt_counts
]
)
prisma.db.litellm_shadowevalattempt.create = AsyncMock()
return prisma
@ -61,6 +69,7 @@ def _job_record(job: ActiveShadowEvalJob, api_key_id="key-hash") -> MagicMock:
shadow_percentage=job.shadow_percentage,
judge_model=job.judge_model,
max_turns=job.max_turns,
max_budget=job.max_budget,
ends_at=job.ends_at,
).items():
setattr(record, field, value)
@ -92,13 +101,32 @@ def _router(shadow_text="shadow answer", judge_json='{"preference": "A", "confid
return router
def _logger(router=None, prisma=None, jobs=()) -> ShadowEvalLogger:
def _spend_counter(store=None):
"""In-memory stand-in for the proxy's cross-pod spend counter: reads take the max of
the counter and the caller's fallback, exactly like get_current_spend does for a key
shape the reseed helpers do not know."""
counter = store if store is not None else {}
async def read(key, fallback_spend, max_budget):
return max(counter.get(key, 0.0), fallback_spend)
async def write(key, cost):
counter[key] = counter.get(key, 0.0) + cost
return counter, read, write
def _logger(router=None, prisma=None, jobs=(), counter_store=None) -> ShadowEvalLogger:
cache = InMemoryCache(max_size_in_memory=4, default_ttl=60)
counter, read, write = _spend_counter(counter_store)
logger = ShadowEvalLogger(
router_provider=lambda: router,
prisma_provider=lambda: prisma,
jobs_cache=cache,
job_spend_reader=read,
job_spend_writer=write,
)
logger._test_counter = counter
if jobs:
cache.set_cache("shadow_eval:active_jobs", {"key-hash": tuple(jobs)})
return logger
@ -447,7 +475,9 @@ def test_failure_detail_names_the_raising_frame():
except TypeError as e:
detail = _failure_detail(e)
lineno = e.__traceback__.tb_lineno
assert detail == f"TypeError at test_shadow_eval_logger.py:{lineno}: 'tuple' object does not support item assignment"
assert (
detail == f"TypeError at test_shadow_eval_logger.py:{lineno}: 'tuple' object does not support item assignment"
)
try:
raise ValueError("p" * 5 * _MAX_ERROR_CHARS)
@ -456,6 +486,73 @@ def test_failure_detail_names_the_raising_frame():
assert "ValueError at test_shadow_eval_logger.py:" in truncated_row_error
def test_call_cost_prefers_the_billed_figure_over_the_public_price_map(monkeypatch):
"""The router client stamps _hidden_params.response_cost from the deployment's own
pricing; the public map reads 0 for deployment-priced models, so budgets gated on it
would never close. The map is only the fallback for responses with no stamp."""
import litellm as litellm_module
from litellm.integrations.shadow_eval_logger import _call_cost
monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005)
stamped = MagicMock()
stamped._hidden_params = {"response_cost": 0.04}
assert _call_cost(stamped) == 0.04
from litellm.types.utils import HiddenParams
object_stamped = MagicMock()
object_stamped._hidden_params = HiddenParams(response_cost=0.03)
assert _call_cost(object_stamped) == 0.03
unstamped = MagicMock()
unstamped._hidden_params = {"response_cost": None}
assert _call_cost(unstamped) == 0.005
assert _call_cost({"choices": []}) == 0.005
@pytest.mark.asyncio
async def test_a_cold_or_reset_counter_degrades_to_the_fill_floor_not_zero(monkeypatch: pytest.MonkeyPatch):
"""The design leans on one owner contract: for a spend:shadow_eval:* key (no DB
reseed by design), get_current_spend returns the caller's fill-sum fallback whenever
the counter reads lower. A reset counter therefore degrades to the <=10s-stale DB
sum, never to zero, so a Redis expiry cannot re-open a spent budget by a full cap."""
from litellm.proxy import proxy_server
counter_key = "spend:shadow_eval:job-cold-test"
monkeypatch.setattr(proxy_server, "prisma_client", None)
proxy_server.spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.05)
try:
assert (
await proxy_server.get_current_spend(counter_key=counter_key, fallback_spend=0.42, max_budget=1.0) == 0.42
)
proxy_server.spend_counter_cache.in_memory_cache.delete_cache(key=counter_key)
assert (
await proxy_server.get_current_spend(counter_key=counter_key, fallback_spend=0.42, max_budget=1.0) == 0.42
)
finally:
proxy_server.spend_counter_cache.in_memory_cache.delete_cache(key=counter_key)
@pytest.mark.asyncio
async def test_an_unverifiable_budget_skips_the_sample_instead_of_spending():
"""A raising spend read (fail-closed enforcement, or an owner bug) must skip the
sample before any provider call, never admit it on a guess."""
async def unverifiable(key, fallback_spend, max_budget):
raise RuntimeError("budget unverifiable")
prisma = _prisma()
router = _router()
logger = _logger(router=router, prisma=prisma, jobs=(_job(max_budget=1.0),))
logger._read_job_spend = unverifiable
await logger.async_log_success_event(_success_kwargs(), RESPONSE, None, None)
await _drain(logger)
router.acompletion.assert_not_called()
prisma.db.litellm_shadowevalattempt.create.assert_not_called()
def test_judge_prompt_is_bounded_however_large_the_inputs():
prompt = _judge_user_prompt("c" * 200_000, "a" * 200_000, "b" * 200_000)
assert len(prompt) < _MAX_JUDGE_PROMPT_CHARS + 100
@ -491,6 +588,7 @@ class TestSuccessHookSkipChain:
assert row["shadow_model"] == "cheap-model"
assert row["confidence"] == 0.9
assert row["judge_cost"] == 0.005
assert row["shadow_cost"] == 0.005
assert row["error"] is None
assert prisma.db.litellm_shadowevaljob.find_many.await_count == 0
@ -580,6 +678,7 @@ class TestSuccessHookSkipChain:
({}, {"ends_at": datetime.now(timezone.utc) - timedelta(seconds=1)}),
({}, {"attempts": 200}),
({}, {"attempts": 199, "max_turns": 200, "_starts": 1}),
({}, {"max_budget": 0.10, "spend": 0.10}),
],
ids=[
"internal-origin",
@ -590,6 +689,7 @@ class TestSuccessHookSkipChain:
"past-end",
"turn-budget-reached",
"budget-consumed-by-started-tasks",
"spend-budget-reached",
],
)
async def test_skip_paths_store_nothing(self, kwargs_mutation, job_mutation):
@ -617,6 +717,61 @@ class TestSuccessHookSkipChain:
assert prisma.db.litellm_shadowevalattempt.create.await_count == 1
async def test_completed_pipelines_hold_spend_budget_within_a_cache_generation(self, monkeypatch):
"""An attempt's recorded cost lands in the spend counter immediately, so the
second sample is skipped before any provider call even though the cached fill
still reads spend 0."""
import litellm as litellm_module
monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005)
prisma = _prisma()
logger = _logger(router=_router(), prisma=prisma, jobs=(_job(max_budget=0.009, spend=0.0),))
await logger.async_log_success_event(_success_kwargs(request_id="req-1"), RESPONSE, None, None)
await _drain(logger)
await logger.async_log_success_event(_success_kwargs(request_id="req-2"), RESPONSE, None, None)
await _drain(logger)
assert prisma.db.litellm_shadowevalattempt.create.await_count == 1
assert logger._test_counter["spend:shadow_eval:job-1"] == 0.01
async def test_a_sibling_pod_sees_spend_through_the_shared_counter(self, monkeypatch):
"""Two pods share the cross-pod counter: once pod A's attempts spend the budget,
pod B skips before its shadow call even though pod B's cached fill reads 0."""
import litellm as litellm_module
monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.005)
shared = {}
prisma_a = _prisma()
pod_a = _logger(
router=_router(), prisma=prisma_a, jobs=(_job(max_budget=0.009, spend=0.0),), counter_store=shared
)
router_b = _router()
prisma_b = _prisma()
pod_b = _logger(
router=router_b, prisma=prisma_b, jobs=(_job(max_budget=0.009, spend=0.0),), counter_store=shared
)
await pod_a.async_log_success_event(_success_kwargs(request_id="req-1"), RESPONSE, None, None)
await _drain(pod_a)
await pod_b.async_log_success_event(_success_kwargs(request_id="req-2"), RESPONSE, None, None)
await _drain(pod_b)
assert prisma_a.db.litellm_shadowevalattempt.create.await_count == 1
prisma_b.db.litellm_shadowevalattempt.create.assert_not_called()
router_b.acompletion.assert_not_called()
async def test_legacy_jobs_without_a_spend_budget_sample_on_turns_alone(self):
"""A pre-migration job carries max_budget None: recorded spend must never gate it,
only its own max_turns can."""
prisma = _prisma()
logger = _logger(router=_router(), prisma=prisma, jobs=(_job(max_budget=None, spend=999.0, attempts=5),))
await logger.async_log_success_event(_success_kwargs(request_id="req-1"), RESPONSE, None, None)
await _drain(logger)
assert prisma.db.litellm_shadowevalattempt.create.await_count == 1
async def test_v1_messages_surface_forwards_identity_from_litellm_metadata(self):
"""/v1/messages stores identity in litellm_params.litellm_metadata, so the hook
resolves the bucket through the shared helper; every surface forwards the same
@ -714,7 +869,7 @@ class TestActiveJobsCache:
async def test_cache_refill_resets_the_starts_counter(self):
job = _job()
prisma = _prisma(jobs=[_job_record(job)], attempt_counts=[("job-1", 7)])
prisma = _prisma(jobs=[_job_record(job)], attempt_counts=[("job-1", 7)], attempt_costs=[("job-1", 0.02, 0.03)])
logger = ShadowEvalLogger(
router_provider=lambda: None,
prisma_provider=lambda: prisma,
@ -722,9 +877,11 @@ class TestActiveJobsCache:
)
logger._job_starts = {"job-1": 5}
await logger._active_jobs()
jobs = await logger._active_jobs()
assert logger._job_starts == {}
assert jobs["key-hash"][0].attempts == 7
assert jobs["key-hash"][0].spend == 0.05
@pytest.mark.asyncio
@ -749,9 +906,9 @@ class TestShadowPipeline:
async def test_over_budget_key_skips_before_any_call(self, monkeypatch: pytest.MonkeyPatch):
"""The gate delegates to the auth path's own budget owner, so an over-budget
verdict there (BudgetExceededError) skips the shadow before any provider call."""
import litellm.proxy.auth.auth_checks as auth_checks
from litellm.exceptions import BudgetExceededError
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth import auth_checks
monkeypatch.setattr(
auth_checks,
@ -777,13 +934,18 @@ class TestShadowPipeline:
prisma.db.litellm_shadowevalattempt.create.assert_not_called()
@pytest.mark.parametrize(
"router_factory,expected_error,expected_cost",
"router_factory,expected_error,expected_cost,expected_shadow_cost",
[
(lambda: _failing_router(), "provider exploded", 0.0),
(lambda: _router(judge_json="I prefer response A, definitely"), "unparseable judge verdict", 0.007),
(lambda: _router(judge_json='{"preference": "'), "unparseable judge verdict", 0.007),
(lambda: _router(judge_json="{}"), "unparseable judge verdict", 0.007),
(lambda: _router(judge_json='{"preference": "A", "confidence": "0.8'), "unparseable judge verdict", 0.007),
(lambda: _failing_router(), "provider exploded", 0.0, 0.0),
(lambda: _router(judge_json="I prefer response A, definitely"), "unparseable judge verdict", 0.007, 0.007),
(lambda: _router(judge_json='{"preference": "'), "unparseable judge verdict", 0.007, 0.007),
(lambda: _router(judge_json="{}"), "unparseable judge verdict", 0.007, 0.007),
(
lambda: _router(judge_json='{"preference": "A", "confidence": "0.8'),
"unparseable judge verdict",
0.007,
0.007,
),
],
ids=[
"shadow-call-fails",
@ -794,7 +956,7 @@ class TestShadowPipeline:
],
)
async def test_failures_become_error_rows_and_keep_billed_judge_cost(
self, router_factory, expected_error, expected_cost, monkeypatch: pytest.MonkeyPatch
self, router_factory, expected_error, expected_cost, expected_shadow_cost, monkeypatch: pytest.MonkeyPatch
):
import litellm as litellm_module
@ -818,6 +980,66 @@ class TestShadowPipeline:
assert expected_error in row["error"]
assert row["confidence"] is None
assert row["judge_cost"] == expected_cost
assert row["shadow_cost"] == expected_shadow_cost
async def test_an_empty_shadow_reply_still_bills_its_cost(self, monkeypatch: pytest.MonkeyPatch):
"""A shadow call that returns no extractable text has still billed; pricing it at
zero would keep the dollar gate open while shadow calls keep charging the key."""
import litellm as litellm_module
monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007)
prisma = _prisma()
logger = _logger(router=_router(shadow_text=""), prisma=prisma)
await logger._run_shadow_eval(
job=_job(),
request_id="req-1",
messages=({"role": "user", "content": "hi"},),
real_text="real answer",
real_model="claude-opus",
control_tier=None,
shadow_params={},
parent_metadata={},
)
row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]
assert row["outcome"] == "error"
assert "empty response" in row["error"]
assert row["shadow_cost"] == 0.007
assert logger._test_counter["spend:shadow_eval:job-1"] == 0.007
async def test_a_pipeline_error_after_the_shadow_call_keeps_its_billed_cost(self, monkeypatch: pytest.MonkeyPatch):
"""An unexpected error between the billed shadow call and the attempt write must
still record the shadow cost, or the per-key dollar gate undercounts forever."""
import litellm as litellm_module
import litellm.integrations.shadow_eval_logger as shadow_eval_module
monkeypatch.setattr(litellm_module, "completion_cost", lambda completion_response: 0.007)
def explode(conversation, response_a, response_b):
raise RuntimeError("judge prompt build failed")
monkeypatch.setattr(shadow_eval_module, "_judge_user_prompt", explode)
prisma = _prisma()
logger = _logger(router=_router(), prisma=prisma)
await logger._run_shadow_eval(
job=_job(),
request_id="req-1",
messages=({"role": "user", "content": "hi"},),
real_text="real answer",
real_model="claude-opus",
control_tier=None,
shadow_params={},
parent_metadata={},
)
row = prisma.db.litellm_shadowevalattempt.create.call_args.kwargs["data"]
assert row["outcome"] == "error"
assert "pipeline error" in row["error"]
assert row["shadow_cost"] == 0.007
assert row["judge_cost"] == 0.0
assert logger._test_counter["spend:shadow_eval:job-1"] == 0.007
async def test_sub_calls_carry_identity_and_origin_but_never_parent_request_state(self):
prisma = _prisma()
@ -918,9 +1140,7 @@ class TestDirection:
router = _router()
logger = _logger(router=router, prisma=prisma, jobs=(_reverse_job(),))
await logger.async_log_success_event(
_success_kwargs(request_metadata=_routed_by()), RESPONSE, None, None
)
await logger.async_log_success_event(_success_kwargs(request_metadata=_routed_by()), RESPONSE, None, None)
await _drain(logger)
assert router.acompletion.call_args_list[0].kwargs["model"] == "baseline-model"
@ -967,9 +1187,7 @@ class TestDirection:
jobs=(_job(id="forward-job", router_name="other-router"), _reverse_job(id="reverse-job")),
)
await logger.async_log_success_event(
_success_kwargs(request_metadata=_routed_by()), RESPONSE, None, None
)
await logger.async_log_success_event(_success_kwargs(request_metadata=_routed_by()), RESPONSE, None, None)
await _drain(logger)
rows = [call.kwargs["data"] for call in prisma.db.litellm_shadowevalattempt.create.call_args_list]

139
tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py
View file

@ -490,7 +490,7 @@ from litellm.proxy.management_endpoints.auto_router_endpoints import (
start_shadow_eval,
stop_shadow_eval_job,
)
from litellm.types.management_endpoints.auto_router_endpoints import StartShadowEvalRequest
from litellm.types.management_endpoints.auto_router_endpoints import SHADOW_EVAL_TURN_VALVE, StartShadowEvalRequest
VIEWER = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, api_key="sk-view", user_id="viewer")
NON_ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-user", user_id="user")
@ -520,6 +520,7 @@ def _leg_record(**overrides: object) -> MagicMock:
"judge_model": "anthropic/claude-sonnet-5",
"shadow_percentage": 10.0,
"max_turns": 200,
"max_budget": None,
"created_at": datetime(2026, 8, 11, tzinfo=timezone.utc),
"ends_at": datetime.now(timezone.utc) + timedelta(days=7),
"stopped_at": None,
@ -549,11 +550,18 @@ def _shadow_prisma(legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-ha
group read that matched on a leg id would come back empty."""
prisma = MagicMock()
prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[_key_record(token) for token in known_keys])
async def execute_raw(sql: str, *params: object):
if "SET stopped_by" in sql:
group = [row for row in stored if row.group_id == params[0]]
counts = {row["job_id"]: row["attempt_count"] for row in prisma.attempt_rows}
sampling = any(row.stopped_at is None and counts.get(row.id, 0) < row.max_turns for row in group)
spends = {row["job_id"]: row["spend"] for row in prisma.attempt_rows}
sampling = any(
row.stopped_at is None
and counts.get(row.id, 0) < row.max_turns
and (row.max_budget is None or spends.get(row.id, 0.0) < row.max_budget)
for row in group
)
window_open = bool(group) and group[0].ends_at > datetime.now(timezone.utc)
claimable = [row for row in group if row.stopped_by is None]
if not (claimable and sampling and window_open):
@ -602,6 +610,7 @@ def _shadow_prisma(legs=(), agg_rows=None, by_leg_rows=None, known_keys=("key-ha
"judge_model",
"shadow_percentage",
"max_turns",
"max_budget",
"created_at",
"ends_at",
"stopped_at",
@ -639,7 +648,7 @@ def _start_request(**overrides: object) -> StartShadowEvalRequest:
"shadow_percentage": 10.0,
"judge_model": "anthropic/claude-sonnet-5",
"duration_days": 7,
"max_turns": 200,
"max_budget": 5.0,
}
payload.update(overrides)
return StartShadowEvalRequest.model_validate(payload)
@ -663,6 +672,9 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp
assert "j.ends_at <= (NOW() AT TIME ZONE 'utc')" in sweep_sql
assert "SET stopped_at = (NOW() AT TIME ZONE 'utc')" in sweep_sql
assert ">= j.max_turns" in sweep_sql
assert "j.max_budget IS NOT NULL" in sweep_sql
assert ">= j.max_budget" in sweep_sql
assert "SUM(a.judge_cost + a.shadow_cost)" in sweep_sql
assert "j.api_key_id = ANY($1::text[])" in sweep_sql
assert sweep_keys == ["key-hash", "key-hash-2"]
prisma.db.litellm_shadowevaljob.create_many.assert_awaited_once()
@ -670,15 +682,17 @@ async def test_start_shadow_eval_writes_one_leg_per_key_in_one_statement(monkeyp
assert [row["api_key_id"] for row in rows] == ["key-hash", "key-hash-2"]
assert len({frozenset((k, v) for k, v in row.items() if k != "api_key_id") for row in rows}) == 1
assert len({row["group_id"] for row in rows}) == 1
assert all(row["max_turns"] == 200 and row["created_by"] == "admin" for row in rows)
assert all(row["max_turns"] == SHADOW_EVAL_TURN_VALVE and row["created_by"] == "admin" for row in rows)
assert all(row["max_budget"] == 5.0 for row in rows)
assert all("status" not in row and "id" not in row for row in rows)
assert response.job_id == rows[0]["group_id"]
assert response.status == "running"
assert response.judged_count is None
assert [(key.api_key_id, key.max_turns, key.key_alias) for key in response.keys] == [
("key-hash", 200, "prod-alpha"),
("key-hash-2", 200, "prod-alpha"),
assert [(key.api_key_id, key.max_budget, key.key_alias) for key in response.keys] == [
("key-hash", 5.0, "prod-alpha"),
("key-hash-2", 5.0, "prod-alpha"),
]
assert all(key.max_turns == SHADOW_EVAL_TURN_VALVE for key in response.keys)
@pytest.mark.asyncio
@ -1041,10 +1055,10 @@ async def test_list_reads_completed_once_every_key_spends_its_budget(monkeypatch
]
)
prisma.attempt_rows = [
{"job_id": "leg-1", "attempt_count": 5},
{"job_id": "leg-2", "attempt_count": 6},
{"job_id": "leg-3", "attempt_count": 5},
{"job_id": "leg-4", "attempt_count": 3},
{"job_id": "leg-1", "attempt_count": 5, "spend": 0.0},
{"job_id": "leg-2", "attempt_count": 6, "spend": 0.0},
{"job_id": "leg-3", "attempt_count": 5, "spend": 0.0},
{"job_id": "leg-4", "attempt_count": 3, "spend": 0.0},
]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
@ -1065,7 +1079,7 @@ async def test_recorded_operator_stop_outranks_budget_arithmetic(monkeypatch: py
stamp = datetime.now(timezone.utc)
prisma = _shadow_prisma(legs=[_leg_record(max_turns=5, stopped_at=stamp, stopped_by="admin")])
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}]
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6, "spend": 0.0}]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
@ -1085,7 +1099,7 @@ async def test_backfilled_legacy_stop_never_reads_as_completion(monkeypatch: pyt
prisma = _shadow_prisma(
legs=[_leg_record(max_turns=5, stopped_at=datetime.now(timezone.utc), stopped_by="unknown")]
)
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6}]
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 6, "spend": 0.0}]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
@ -1108,12 +1122,39 @@ def test_stopped_by_migration_backfills_every_job_that_displayed_stopped():
assert "WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc')" in sql
def test_a_start_request_still_sending_max_turns_is_rejected_not_silently_defaulted():
"""Pydantic ignores unknown fields, so without the explicit rejection a caller still
sending the retired turn budget would silently run on the default dollar budget."""
with pytest.raises(ValidationError, match="max_budget"):
_start_request(max_turns=200)
def test_max_budget_migration_is_additive_and_leaves_legacy_rows_null():
"""max_budget stays NULL on pre-migration rows so they keep the turn budget they were
configured with, and shadow_cost defaults to 0 so old rows price as judge-only."""
import litellm_proxy_extras
sql = (
Path(litellm_proxy_extras.__file__).parent
/ "migrations"
/ "20260819000000_shadow_eval_max_budget"
/ "migration.sql"
).read_text()
assert 'ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN "max_budget" DOUBLE PRECISION' in sql
assert (
'ALTER TABLE "LiteLLM_ShadowEvalAttempt" ADD COLUMN "shadow_cost" DOUBLE PRECISION NOT NULL DEFAULT 0'
in sql
)
assert "UPDATE" not in sql
assert "DROP" not in sql
@pytest.mark.asyncio
async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(legs=[_leg_record(max_turns=3)])
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 3}]
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 3, "spend": 0.0}]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
with pytest.raises(HTTPException) as exhausted:
@ -1123,6 +1164,71 @@ async def test_stop_rejects_a_job_that_already_spent_its_budget(monkeypatch: pyt
prisma.db.litellm_shadowevaljob.update_many.assert_not_called()
@pytest.mark.asyncio
async def test_list_reads_completed_once_every_key_spends_its_dollar_budget(monkeypatch: pytest.MonkeyPatch):
"""A spend-budgeted job completes on dollars, not turns: every key's recorded shadow
plus judge spend reaching max_budget reads completed long before the turn valve, while
one key with budget left keeps the whole job running."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(
legs=[
_leg_record(max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0),
_leg_record(id="leg-2", api_key_id="key-hash-2", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0),
_leg_record(
id="leg-3", group_id="job-2", api_key_id="key-hash", max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=1.0
),
]
)
prisma.attempt_rows = [
{"job_id": "leg-1", "attempt_count": 40, "spend": 1.0},
{"job_id": "leg-2", "attempt_count": 55, "spend": 1.25},
{"job_id": "leg-3", "attempt_count": 40, "spend": 0.99},
]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
by_id = {job.job_id: job for job in jobs}
assert by_id["job-1"].status == "completed"
assert by_id["job-2"].status == "running"
assert {key.api_key_id: key.spend for key in by_id["job-1"].keys} == {"key-hash": 1.0, "key-hash-2": 1.25}
assert all(key.max_budget == 1.0 for key in by_id["job-1"].keys)
@pytest.mark.asyncio
async def test_stop_rejects_a_job_whose_dollar_budget_is_spent(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(legs=[_leg_record(max_turns=SHADOW_EVAL_TURN_VALVE, max_budget=0.5)])
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 7, "spend": 0.5}]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
with pytest.raises(HTTPException) as exhausted:
await stop_shadow_eval_job("job-1", ADMIN)
assert exhausted.value.status_code == 400
assert "completed" in exhausted.value.detail
prisma.db.litellm_shadowevaljob.update_many.assert_not_called()
@pytest.mark.asyncio
async def test_legacy_jobs_without_a_dollar_budget_stay_turn_gated(monkeypatch: pytest.MonkeyPatch):
"""A job from before spend budgets existed carries max_budget NULL: recorded spend
can never complete it, only its own max_turns can, so migration changes nothing about
what it was configured to do."""
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(legs=[_leg_record(max_turns=200, max_budget=None)])
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 40, "spend": 250.0}]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
jobs = await list_shadow_eval_jobs(VIEWER, api_key_id=None, limit=50)
assert jobs[0].status == "running"
assert jobs[0].keys[0].max_budget is None
assert jobs[0].keys[0].spend == 250.0
@pytest.mark.asyncio
async def test_shadow_eval_responses_name_every_shadowed_key(monkeypatch: pytest.MonkeyPatch):
import litellm.proxy.proxy_server as proxy_server
@ -1167,6 +1273,9 @@ async def test_stop_shadow_eval_stops_every_unstopped_leg_and_rejects_non_runnin
assert "WHERE group_id = $1 AND stopped_by IS NULL" in stop_sql
assert "ends_at > (NOW() AT TIME ZONE 'utc')" in stop_sql
assert ") < k.max_turns" in stop_sql
assert "k.max_budget IS NULL" in stop_sql
assert ") < k.max_budget" in stop_sql
assert "SUM(a.judge_cost + a.shadow_cost)" in stop_sql
assert (stop_group, stop_operator) == ("job-1", "admin")
assert datetime.fromisoformat(stop_stamp).tzinfo is None
assert prisma.db.execute_raw.await_count == 1
@ -1357,7 +1466,7 @@ async def test_a_stop_racing_the_last_budgeted_attempt_reports_completed_not_sto
import litellm.proxy.proxy_server as proxy_server
prisma = _shadow_prisma(legs=[_leg_record(max_turns=2)])
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 2}]
prisma.attempt_rows = [{"job_id": "leg-1", "attempt_count": 2, "spend": 0.0}]
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
with pytest.raises(HTTPException) as exc:

52
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.test.tsx
View file

@ -80,7 +80,9 @@ const job = (overrides: Partial<ShadowEvalJob> = {}): ShadowEvalJob => ({
keys: [
{
api_key_id: "hashed-key-abc",
max_turns: 200,
max_turns: 10000,
max_budget: 10,
spend: 3.21,
stopped_at: null,
key_alias: "prod-alpha",
key_name: "sk-...alpha",
@ -133,7 +135,9 @@ const keyEntry = (
overrides: Partial<ShadowEvalJob["keys"][number]> = {},
): ShadowEvalJob["keys"][number] => ({
api_key_id,
max_turns: 200,
max_turns: 10000,
max_budget: 10,
spend: 0,
stopped_at: null,
attempt_count: null,
key_alias: null,
@ -321,6 +325,21 @@ describe("ShadowEvalSection", () => {
expect(screen.getByText(/ends in 3 days/)).toBeInTheDocument();
});
it("shows recorded eval spend against the job's dollar budget", () => {
const j = job();
mockHooks({ jobs: [j], detailsById: { "job-1": j } });
render(<ShadowEvalSection />);
expect(screen.getByText(/\$3\.21 of \$10\.00 eval spend/)).toBeInTheDocument();
});
it("shows spend without a budget cap for a job from before spend budgets existed", () => {
const j = job({ keys: [keyEntry("hashed-key-abc", { max_budget: null, spend: 3.21 })] });
mockHooks({ jobs: [j], detailsById: { "job-1": j } });
render(<ShadowEvalSection />);
expect(screen.getByText(/\$3\.21 eval spend/)).toBeInTheDocument();
expect(screen.queryByText(/of \$/)).not.toBeInTheDocument();
});
it("flags rows with fewer than 30 judged turns as low sample", () => {
const j = job();
mockHooks({ jobs: [j], detailsById: { "job-1": j } });
@ -388,7 +407,7 @@ describe("ShadowEvalSection", () => {
direction: "forward",
shadow_percentage: 10,
duration_days: 7,
max_turns: 200,
max_budget: 10,
judge_model: "anthropic/claude-sonnet-5",
};
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
@ -425,7 +444,7 @@ describe("ShadowEvalSection", () => {
baseline_model: "prod-claude",
shadow_percentage: 10,
duration_days: 7,
max_turns: 200,
max_budget: 10,
judge_model: "anthropic/claude-sonnet-5",
};
expect(start.mutate).toHaveBeenCalledWith(expectedBody);
@ -463,8 +482,8 @@ describe("ShadowEvalSection", () => {
job({
judged_count: 205,
keys: [
keyEntry("hash-spent", { max_turns: 200, stopped_at: "2026-08-08T00:00:00Z" }),
keyEntry("hash-hungry", { max_turns: 500 }),
keyEntry("hash-spent", { max_budget: 2, spend: 1.5, stopped_at: "2026-08-08T00:00:00Z" }),
keyEntry("hash-hungry", { max_budget: 5, spend: 0.2 }),
],
results: {
by_tier: [],
@ -492,25 +511,26 @@ describe("ShadowEvalSection", () => {
if (!spent || !hungry) throw new Error("expected a table row per scoped key");
expect(within(spent).getByText("stopped")).toBeInTheDocument();
expect(within(spent).getByText("200 / 200")).toBeInTheDocument();
expect(within(spent).getByText("$1.50 / $2.00")).toBeInTheDocument();
expect(within(spent).getByText("60.0%")).toBeInTheDocument();
expect(within(hungry).getByText("running")).toBeInTheDocument();
expect(within(hungry).getByText("0 / 500")).toBeInTheDocument();
expect(within(hungry).getByText("$0.2000 / $5.00")).toBeInTheDocument();
expect(within(hungry).getByText("No verdicts yet")).toBeInTheDocument();
expect(screen.getByText(/205 of 700 turns judged/)).toBeInTheDocument();
expect(screen.getByText(/205 turns judged/)).toBeInTheDocument();
expect(screen.getByText(/Shadowing 10% of/)).toBeInTheDocument();
expect(screen.getByText("2 keys")).toBeInTheDocument();
});
it("reads a key that spent its budget as completed even before the sweep stamps it", () => {
const legacyTurnBudgetLeg = { max_budget: null, spend: 0.5, max_turns: 500, attempt_count: 3 };
mockHooks({
jobs: [
job({
keys: [
keyEntry("hash-spent", { max_turns: 200, attempt_count: 200 }),
keyEntry("hash-hungry", { max_turns: 500, attempt_count: 3 }),
keyEntry("hash-spent", { max_budget: 2, spend: 2, attempt_count: 40 }),
keyEntry("hash-hungry", legacyTurnBudgetLeg),
],
}),
],
@ -521,9 +541,9 @@ describe("ShadowEvalSection", () => {
const hungry = screen.getByText("hash-hungr…").closest("tr");
if (!spent || !hungry) throw new Error("expected a table row per scoped key");
expect(within(spent).getByText("completed")).toBeInTheDocument();
expect(within(spent).getByText("200 / 200")).toBeInTheDocument();
expect(within(spent).getByText("$2.00 / $2.00")).toBeInTheDocument();
expect(within(hungry).getByText("running")).toBeInTheDocument();
expect(within(hungry).getByText("3 / 500")).toBeInTheDocument();
expect(within(hungry).getByText("3 / 500 turns")).toBeInTheDocument();
});
it("shows the per-key table while a multi-key job is still collecting, before any verdicts exist", () => {
@ -533,8 +553,8 @@ describe("ShadowEvalSection", () => {
judged_count: 0,
results: null,
keys: [
keyEntry("hash-spent", { max_turns: 2, attempt_count: 2 }),
keyEntry("hash-hungry", { max_turns: 500, attempt_count: 1 }),
keyEntry("hash-spent", { max_budget: 0.5, spend: 0.5, attempt_count: 2 }),
keyEntry("hash-hungry", { max_budget: 5, spend: 0.01, attempt_count: 1 }),
],
}),
],
@ -544,7 +564,7 @@ describe("ShadowEvalSection", () => {
const spent = screen.getByText("hash-spent…").closest("tr");
if (!spent) throw new Error("expected a per-key row before verdicts exist");
expect(within(spent).getByText("completed")).toBeInTheDocument();
expect(within(spent).getByText("2 / 2")).toBeInTheDocument();
expect(within(spent).getByText("$0.5000 / $0.5000")).toBeInTheDocument();
expect(screen.getByText("Budget used")).toBeInTheDocument();
expect(screen.queryByText("Judged turns")).not.toBeInTheDocument();
expect(screen.getByText(/Collecting verdicts/)).toBeInTheDocument();

57
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/ShadowEvalSection.tsx
View file

@ -57,9 +57,19 @@ export const shadowedKeyLabel = (key: ShadowEvalJobKey): string =>
const shadowedKeysLabel = (job: ShadowEvalJob): string =>
job.keys.length === 1 ? shadowedKeyLabel(job.keys[0]) : `${job.keys.length} keys`;
const totalBudget = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + key.max_turns, 0);
const totalBudget = (job: ShadowEvalJob): number | null =>
job.keys.reduce<number | null>(
(sum, key) => (sum === null || key.max_budget == null ? null : sum + key.max_budget),
0,
);
const keySpent = (key: ShadowEvalJobKey): boolean => key.attempt_count != null && key.attempt_count >= key.max_turns;
const totalSpend = (job: ShadowEvalJob): number => job.keys.reduce((sum, key) => sum + (key.spend ?? 0), 0);
const keySpent = (key: ShadowEvalJobKey): boolean => {
const spendBudgetReached = key.max_budget != null && key.spend != null && key.spend >= key.max_budget;
const turnValveReached = key.attempt_count != null && key.attempt_count >= key.max_turns;
return spendBudgetReached || turnValveReached;
};
const keyStatus = (job: ShadowEvalJob, key: ShadowEvalJobKey): string => {
if (job.status === "completed" || (key.stopped_at == null && keySpent(key))) return "completed";
@ -207,7 +217,9 @@ const KeyTable: React.FC<{ job: ShadowEvalJob }> = ({ job }) => {
<StatusBadge status={keyStatus(job, key)} />
</TableCell>
<TableCell className="text-right tabular-nums">
{(key.attempt_count ?? slice?.turn_count ?? 0).toLocaleString()} / {key.max_turns.toLocaleString()}
{key.max_budget != null
? `${usd(key.spend ?? 0)} / ${usd(key.max_budget)}`
: `${(key.attempt_count ?? slice?.turn_count ?? 0).toLocaleString()} / ${key.max_turns.toLocaleString()} turns`}
</TableCell>
{slice ? (
<>
@ -300,8 +312,9 @@ const JobResults: React.FC<{
<div>
<p className="text-sm font-medium text-foreground">{jobHeadline(job)}</p>
<p className="text-xs text-muted-foreground">
{(job.judged_count ?? 0).toLocaleString()} of {totalBudget(job).toLocaleString()} turns judged ·{" "}
{(job.error_count ?? 0).toLocaleString()} errored · {usd(job.judge_spend ?? 0)} judge spend
{(job.judged_count ?? 0).toLocaleString()} turns judged · {(job.error_count ?? 0).toLocaleString()}{" "}
errored · {usd(totalSpend(job))}
{totalBudget(job) !== null ? ` of ${usd(totalBudget(job) ?? 0)}` : ""} eval spend
{active && remaining ? ` · ${remaining}` : ""}
</p>
</div>
@ -375,9 +388,9 @@ const DIRECTION_OPTIONS: readonly { value: ShadowEvalDirection; label: string }[
const START_FORM_DESCRIPTION: Record<ShadowEvalDirection, string> = {
forward:
"Duplicates a sampled slice of the selected keys' traffic through the auto-router and has an LLM judge compare both answers blind. Each key gets its own turn budget. The router's answers are never served to users; judge calls bill to the shadowed key.",
"Duplicates a sampled slice of the selected keys' traffic through the auto-router and has an LLM judge compare both answers blind. Each key gets its own spend budget. The router's answers are never served to users; judge calls bill to the shadowed key.",
reverse:
"Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each key gets its own turn budget. The baseline's answers are never served to users; judge calls bill to the shadowed key.",
"Duplicates a sampled slice of the traffic the auto-router already serves against a fixed baseline model and has an LLM judge compare both answers blind. Each key gets its own spend budget. The baseline's answers are never served to users; judge calls bill to the shadowed key.",
};
const DURATION_OPTIONS = [
@ -445,7 +458,7 @@ const StartForm: React.FC = () => {
const [percentage, setPercentage] = useState("10");
const [durationDays, setDurationDays] = useState("7");
const [judgeModel, setJudgeModel] = useState("");
const [maxTurns, setMaxTurns] = useState("200");
const [maxBudget, setMaxBudget] = useState("10");
const { data: autoRouters } = useAutoRouters();
const judgeModelOptions = useJudgeModelOptions();
const baselineModelOptions = useBaselineModelOptions();
@ -460,11 +473,11 @@ const StartForm: React.FC = () => {
const parsedPct = Number.parseFloat(percentage);
const percentageValid = parsedPct >= 0.1 && parsedPct <= 100;
const parsedMaxTurns = Number.parseInt(maxTurns, 10);
const maxTurnsValid = parsedMaxTurns >= 1 && parsedMaxTurns <= 2000;
const parsedMaxBudget = Number.parseFloat(maxBudget);
const maxBudgetValid = parsedMaxBudget >= 0.01 && parsedMaxBudget <= 10000;
const baselinePicked = direction === "forward" || baselineModel !== "";
const filled = apiKeyIds.length > 0 && [routerName, judgeModel].every((field) => field !== "") && baselinePicked;
const boundsValid = percentageValid && maxTurnsValid;
const boundsValid = percentageValid && maxBudgetValid;
const valid = Boolean(accessToken) && filled && boundsValid;
const handleStart = () => {
const startBody = {
@ -474,7 +487,7 @@ const StartForm: React.FC = () => {
...(direction === "reverse" ? { baseline_model: baselineModel } : {}),
shadow_percentage: parsedPct,
duration_days: Number.parseInt(durationDays, 10),
max_turns: parsedMaxTurns,
max_budget: parsedMaxBudget,
judge_model: judgeModel,
};
start.mutate(startBody);
@ -551,20 +564,22 @@ const StartForm: React.FC = () => {
</SelectContent>
</Select>
</Field>
<Field label="Turn budget">
<Field label="Spend budget">
<div className="flex items-center gap-2">
<span className="text-sm text-muted-foreground">$</span>
<Input
type="number"
min={1}
max={2000}
min={0.01}
max={10000}
step={0.01}
className="w-24"
value={maxTurns}
onChange={(e) => setMaxTurns(e.target.value)}
value={maxBudget}
onChange={(e) => setMaxBudget(e.target.value)}
/>
<span className="text-sm text-muted-foreground">turns judged, max</span>
<span className="text-sm text-muted-foreground">max shadow + judge spend, per key</span>
</div>
{maxTurns.trim() !== "" && !maxTurnsValid && (
<p className="text-xs text-destructive">Enter a value from 1 to 2000</p>
{maxBudget.trim() !== "" && !maxBudgetValid && (
<p className="text-xs text-destructive">Enter a value from 0.01 to 10000</p>
)}
</Field>
{direction === "reverse" && (
@ -620,7 +635,7 @@ const PreviousJob: React.FC<{ job: ShadowEvalJob }> = ({ job }) => {
<p className="text-sm font-medium text-foreground">{jobHeadline(shown)}</p>
<p className="text-xs text-muted-foreground">
{shown.judged_count != null &&
`${shown.judged_count.toLocaleString()} judged · ${(shown.error_count ?? 0).toLocaleString()} errored · ${usd(shown.judge_spend ?? 0)} judge spend · `}
`${shown.judged_count.toLocaleString()} judged · ${(shown.error_count ?? 0).toLocaleString()} errored · ${usd(totalSpend(shown))} eval spend · `}
{new Date(shown.created_at).toLocaleDateString()}
</p>
</div>

33
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -849,11 +849,12 @@ export interface paths {
* serve and duplicates them against baseline_model. A key can hold one active job per
* direction, so both questions can run at once.
*
* Shadow responses are never served to users. Each key samples until it has judged
* max_turns turns of its own traffic, the job's window ends, or the job is stopped, so one
* key running out of budget does not end sampling for the others; sampling changes
* propagate to pods within about 10 seconds. Shadow and judge calls bill to the shadowed
* key but are excluded from request counts and auto-router adoption metrics.
* Shadow responses are never served to users. Each key samples until its recorded eval
* spend, the shadow and judge calls' own cost, reaches max_budget dollars, the job's
* window ends, or the job is stopped, so one key running out of budget does not end
* sampling for the others; sampling changes propagate to pods within about 10 seconds.
* Shadow and judge calls bill to the shadowed key but are excluded from request counts
* and auto-router adoption metrics.
*/
post: operations["start_shadow_eval_auto_router_shadow_eval_start_post"];
delete?: never;
@ -33305,11 +33306,21 @@ export interface components {
* @description Masked display name (sk-...) of the shadowed key, resolved at read time like key_alias
*/
key_name?: string | null;
/**
* Max Budget
* @description This key's own USD budget for the eval's shadow and judge spend, independent of its siblings'; None on jobs created before spend budgets existed, which max_turns alone bounds
*/
max_budget?: number | null;
/**
* Max Turns
* @description This key's own sample budget, independent of its siblings'
* @description This key's sample-count ceiling: the whole budget for jobs created before max_budget existed, and the error-loop safety valve otherwise
*/
max_turns: number;
/**
* Spend
* @description This key's recorded shadow plus judge spend in USD, the same figure the sampler budgets against max_budget; populated on list and detail responses and frozen at stopped_at exactly like attempt_count
*/
spend?: number | null;
/**
* Stopped At
* @description When this key's slot was stamped free, whether its own budget ran out, the window closed, or an operator stopped the job; status is derived, so a spent budget reads completed even while this is still unset
@ -33626,7 +33637,7 @@ export interface components {
StartShadowEvalRequest: {
/**
* Api Key Ids
* @description The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these keys' traffic; requests made with any other key are not sampled. Each key carries its own max_turns budget, so one key exhausting its budget leaves the others sampling. At most 100 keys per job, which also bounds every read the job's endpoints make.
* @description The hashed virtual keys whose traffic will be shadowed. Shadow evaluation runs ONLY on these keys' traffic; requests made with any other key are not sampled. Each key carries its own max_budget spend budget, so one key exhausting its budget leaves the others sampling. At most 100 keys per job, which also bounds every read the job's endpoints make.
*/
api_key_ids: string[];
/**
@ -33654,11 +33665,11 @@ export interface components {
*/
judge_model: string;
/**
* Max Turns
* @description Per-key sample budget: the job judges at most this many turns of EACH scoped key's traffic, so a job over N keys judges at most N times max_turns turns. This is also the spend bound; expected judge cost is roughly that turn ceiling times one judge call
* @default 200
* Max Budget
* @description Per-key USD budget for the eval's own overhead, the shadow-arm and judge calls, priced with the same figures the spend pipeline bills. EACH scoped key samples until its recorded eval spend reaches this, so a job over N keys spends at most about N times max_budget; in-flight samples can overshoot the cap by one sampling cache window
* @default 10
*/
max_turns: number;
max_budget: number;
/**
* Router Name
* @description The auto-router under evaluation, in either direction